Add scrape/ingestion engine + validation, and 24h shift hour-range visualization

Scrape engine (Services/Scraping/): pluggable IListingSource (working sample + Telegram/Divar credential-ready stubs) → IngestionService (content-hash dedupe → parse → validate → review queue) → ListingValidator (completeness score + spam screen) → IngestionWorker (config-gated hosted service). RawListing gains ContentHash/Confidence/ValidationNotes; RawListingStatus.Flagged. Admin /Admin gets run-now, source list, confidence + flagged queue. Hour-range viz: _HourBar 24h timeline bar (colored by type, overnight wrap) on shift cards, recommendation cards, and detail. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-03 08:18:19 +03:30
parent 69fa921fbd
commit 931b7b6ffb
24 changed files with 1439 additions and 26 deletions
@@ -6,29 +6,55 @@

 <div class="page-head">
    <div class="container">
-        <h1>پنل مدیریت — صف آگهی‌های خام</h1>
+        <h1>پنل مدیریت — جمع‌آوری و صف آگهی‌ها</h1>
        <p class="muted">
-            آگهی‌های جمع‌آوری‌شده از کانال‌ها را اینجا بررسی، ساختارمند و منتشر کن.
-            (@JalaliDate.ToPersianDigits(Model.Queue.Count.ToString()) در انتظار بررسی)
+            آگهی‌های جمع‌آوری‌شده از منابع را بررسی، ساختارمند و منتشر کن.
+            (@JalaliDate.ToPersianDigits(Model.Queue.Count.ToString()) در صف،
+            @JalaliDate.ToPersianDigits(Model.Flagged.Count.ToString()) پرچم‌خورده)
            · <a asp-page="/Admin/Facilities">تأیید مراکز درمانی</a>
        </p>
    </div>
 </div>

 <div class="container section">
+    @if (Model.IngestMessage is not null)
+    {
+        <div class="alert alert-success">✓ @Model.IngestMessage</div>
+    }
+
    <div class="layout-2">
        <aside class="card card-pad filter-card">
-            <h3>افزودن آگهی خام</h3>
+            <h3>موتور جمع‌آوری</h3>
+            <p class="muted" style="font-size:13px;">منابع متصل:</p>
+            <ul style="margin:0 0 12px; padding-inline-start:18px; font-size:13.5px;">
+                @foreach (var src in Model.Sources)
+                {
+                    <li>@src.Name —
+                        @if (src.Enabled) { <span style="color:var(--primary-dark);">فعال</span> }
+                        else { <span class="muted">غیرفعال (نیازمند تنظیمات)</span> }
+                    </li>
+                }
+            </ul>
+            <form method="post">
+                <button type="submit" asp-page-handler="RunIngestion" class="btn btn-accent btn-block">اجرای جمع‌آوری اکنون</button>
+            </form>
+            <p class="muted" style="font-size:11px; margin:8px 0 0;">
+                موتور: واکشی ← حذف تکراری ← تجزیه ← اعتبارسنجی ← صف بررسی.
+            </p>
+
+            <hr style="border:none; border-top:1px solid var(--line); margin:16px 0;" />
+
+            <h3>افزودن دستی</h3>
            <form method="post">
                <div class="filter-group">
-                    <label>منبع (کانال/سایت)</label>
+                    <label>منبع</label>
                    <input type="text" name="SourceChannel" placeholder="مثلاً کانال شیفت تهران" />
                </div>
                <div class="filter-group">
                    <label>متن آگهی</label>
-                    <textarea name="RawText" rows="6" placeholder="متن کپی‌شده از تلگرام/بله/دیوار را اینجا بچسبان..."></textarea>
+                    <textarea name="RawText" rows="5" placeholder="متن کپی‌شده را بچسبان..."></textarea>
                </div>
-                <button type="submit" asp-page-handler="Add" class="btn btn-primary btn-block">افزودن به صف</button>
+                <button type="submit" asp-page-handler="Add" class="btn btn-outline btn-block">افزودن به صف</button>
            </form>
            <p class="muted" style="font-size:12px; margin-bottom:0;">
                منتشرشده: @JalaliDate.ToPersianDigits(Model.PublishedShifts.ToString()) شیفت،
@@ -37,22 +63,26 @@
        </aside>

        <div>
+            <h2 style="font-size:20px; margin-top:0;">صف بررسی</h2>
            @if (Model.Queue.Count == 0)
            {
-                <div class="card empty-state">صف خالی است. آگهی جدیدی برای بررسی وجود ندارد.</div>
+                <div class="card empty-state">صف خالی است. «اجرای جمع‌آوری» را بزن یا آگهی اضافه کن.</div>
            }
            else
            {
                foreach (var r in Model.Queue)
                {
-                    <div class="card card-pad" style="margin-bottom:14px;">
-                        <div class="row" style="display:flex; justify-content:space-between;">
-                            <strong>@r.SourceChannel</strong>
-                            <span class="muted" style="font-size:12px;">@JalaliDate.ToLongDate(DateOnly.FromDateTime(r.FetchedAt))</span>
-                        </div>
-                        <p style="margin:10px 0; white-space:pre-wrap;">@r.RawText</p>
-                        <a class="btn btn-accent" asp-page="/Admin/Review" asp-route-id="@r.Id">بررسی و انتشار ←</a>
-                    </div>
+                    <partial name="_RawListingRow" model="r" />
+                }
+            }
+
+            @if (Model.Flagged.Count > 0)
+            {
+                <h2 style="font-size:20px; margin-top:28px;">پرچم‌خورده (ناقص/مشکوک)</h2>
+                <p class="muted" style="font-size:13px;">اعتبارسنجی این‌ها را کامل ندانست؛ در صورت صحت می‌توانی منتشرشان کنی.</p>
+                foreach (var r in Model.Flagged)
+                {
+                    <partial name="_RawListingRow" model="r" />
                }
            }
        </div>
@@ -1,5 +1,6 @@
 using JobsMedical.Web.Data;
 using JobsMedical.Web.Models;
+using JobsMedical.Web.Services.Scraping;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.RazorPages;
@@ -7,19 +8,29 @@ using Microsoft.EntityFrameworkCore;

 namespace JobsMedical.Web.Pages.Admin;

-[Authorize(Roles = "Admin")] // secured by the OTP-auth Admin role
+[Authorize(Roles = "Admin")]
 public class IndexModel : PageModel
 {
    private readonly AppDbContext _db;
-    public IndexModel(AppDbContext db) => _db = db;
+    private readonly IngestionService _ingest;
+
+    public IndexModel(AppDbContext db, IngestionService ingest)
+    {
+        _db = db;
+        _ingest = ingest;
+    }

    public List<RawListing> Queue { get; private set; } = new();
+    public List<RawListing> Flagged { get; private set; } = new();
+    public IReadOnlyList<(string Name, bool Enabled)> Sources { get; private set; } = new List<(string, bool)>();
    public int PublishedShifts { get; private set; }
    public int PublishedJobs { get; private set; }

    [BindProperty] public string? SourceChannel { get; set; }
    [BindProperty] public string? RawText { get; set; }

+    [TempData] public string? IngestMessage { get; set; }
+
    public async Task OnGetAsync() => await LoadAsync();

    public async Task<IActionResult> OnPostAddAsync()
@@ -37,11 +48,23 @@ public class IndexModel : PageModel
        return RedirectToPage();
    }

+    public async Task<IActionResult> OnPostRunIngestionAsync()
+    {
+        var s = await _ingest.RunAsync();
+        IngestMessage = $"جمع‌آوری انجام شد — {s.TotalQueued} در صف، {s.TotalFlagged} پرچم‌خورده، " +
+                        $"{s.TotalSpam} اسپم، {s.TotalDuplicates} تکراری.";
+        return RedirectToPage();
+    }
+
    private async Task LoadAsync()
    {
        Queue = await _db.RawListings
            .Where(r => r.Status == RawListingStatus.New)
+            .OrderByDescending(r => r.Confidence).ThenByDescending(r => r.FetchedAt).ToListAsync();
+        Flagged = await _db.RawListings
+            .Where(r => r.Status == RawListingStatus.Flagged)
            .OrderByDescending(r => r.FetchedAt).ToListAsync();
+        Sources = _ingest.Sources;
        PublishedShifts = await _db.Shifts.CountAsync(s => s.Source != ShiftSource.Direct);
        PublishedJobs = await _db.JobOpenings.CountAsync();
    }
@@ -0,0 +1,42 @@
+@model JobsMedical.Web.Models.Shift
+@using System.Globalization
+@{
+    var s = Model;
+    var ci = CultureInfo.InvariantCulture;
+    int sm = s.StartTime.Hour * 60 + s.StartTime.Minute;
+    int em = s.EndTime.Hour * 60 + s.EndTime.Minute;
+    var typeClass = s.ShiftType switch
+    {
+        ShiftType.Day => "day",
+        ShiftType.Evening => "evening",
+        ShiftType.Night => "night",
+        _ => "oncall",
+    };
+
+    // Build one or two segments (overnight shifts wrap past midnight). On-call = whole day.
+    var segs = new List<(double left, double width)>();
+    if (s.ShiftType == ShiftType.OnCall || em == sm)
+        segs.Add((0, 100));
+    else if (em > sm)
+        segs.Add((sm / 1440.0 * 100, (em - sm) / 1440.0 * 100));
+    else
+    {
+        segs.Add((sm / 1440.0 * 100, (1440 - sm) / 1440.0 * 100));
+        segs.Add((0, em / 1440.0 * 100));
+    }
+    string Pct(double v) => v.ToString("0.##", ci);
+}
+<div class="hourbar-wrap" title="@JalaliDate.Time(s.StartTime) تا @JalaliDate.Time(s.EndTime)">
+    <div class="hourbar">
+        <span class="hourbar-grid" style="left:25%"></span>
+        <span class="hourbar-grid" style="left:50%"></span>
+        <span class="hourbar-grid" style="left:75%"></span>
+        @foreach (var seg in segs)
+        {
+            <span class="hourbar-fill @typeClass" style="left:@Pct(seg.left)%; width:@Pct(seg.width)%"></span>
+        }
+    </div>
+    <div class="hourbar-axis">
+        <span>۰</span><span>۶</span><span>۱۲</span><span>۱۸</span><span>۲۴</span>
+    </div>
+</div>
@@ -0,0 +1,20 @@
+@model JobsMedical.Web.Models.RawListing
+@{
+    var c = Model.Confidence;
+    var confClass = c >= 70 ? "badge-verified" : c >= 50 ? "badge-day" : "badge-type";
+}
+<div class="card card-pad" style="margin-bottom:12px;">
+    <div class="row" style="display:flex; justify-content:space-between; align-items:center; gap:8px; flex-wrap:wrap;">
+        <strong>@Model.SourceChannel</strong>
+        <span style="display:flex; gap:8px; align-items:center;">
+            <span class="badge @confClass">اطمینان @JalaliDate.ToPersianDigits(c.ToString())٪</span>
+            <span class="muted" style="font-size:12px;">@JalaliDate.ToLongDate(DateOnly.FromDateTime(Model.FetchedAt))</span>
+        </span>
+    </div>
+    <p style="margin:10px 0; white-space:pre-wrap;">@Model.RawText</p>
+    @if (!string.IsNullOrEmpty(Model.ValidationNotes))
+    {
+        <p class="muted" style="font-size:12.5px; margin:0 0 10px;">⚠ @Model.ValidationNotes</p>
+    }
+    <a class="btn btn-accent" asp-page="/Admin/Review" asp-route-id="@Model.Id">بررسی و انتشار ←</a>
+</div>
@@ -22,6 +22,7 @@
        <span>📍 @s.Facility?.City?.Name</span>
    </div>
    <div class="row">📅 @JalaliDate.WeekDayName(s.Date)، @JalaliDate.ToLongDate(s.Date) — 🕐 @JalaliDate.Time(s.StartTime)</div>
+    <partial name="_HourBar" model="s" />

    @* The "why" — what makes a pattern engine trustworthy: every pick is explained. *@
    <div class="rec-reasons">
@@ -30,6 +30,7 @@
    }
    <div class="row">📅 @JalaliDate.WeekDayName(Model.Date)، @JalaliDate.ToLongDate(Model.Date)</div>
    <div class="row">🕐 @JalaliDate.Time(Model.StartTime) تا @JalaliDate.Time(Model.EndTime)</div>
+    <partial name="_HourBar" model="Model" />
    <div class="foot">
        <span class="pay">@JalaliDate.PayLabel(Model.PayType, Model.PayAmount, Model.SharePercent)</span>
        <span class="btn btn-outline" style="padding: 6px 14px;">جزئیات</span>
@@ -50,6 +50,10 @@
                <div class="info-row"><span class="k">مدت</span><span class="v">@JalaliDate.ToPersianDigits(s.DurationHours.ToString("0.#")) ساعت</span></div>
                <div class="info-row"><span class="k">نقش مورد نیاز</span><span class="v">@(s.Role?.Name ?? s.SpecialtyRequired)</span></div>
                <div class="info-row"><span class="k">پرداخت</span><span class="v" style="color:var(--primary-dark)">@JalaliDate.PayLabel(s.PayType, s.PayAmount, s.SharePercent)</span></div>
+                <div style="padding-top:12px;">
+                    <span class="k" style="font-size:13px; color:var(--muted);">بازه ساعت کاری در شبانه‌روز</span>
+                    <partial name="_HourBar" model="s" />
+                </div>
            </div>

            @if (!string.IsNullOrEmpty(s.Description))