From 487c7ca82f5b113fed14de29ab75f365080f4dc5 Mon Sep 17 00:00:00 2001 From: "soroush.asadi" Date: Mon, 8 Jun 2026 06:23:58 +0330 Subject: [PATCH] [Ingest] Persistent crawl run-log + per-source breakdown on admin queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each ingestion run now records an IngestionRun row (found/queued/published/flagged/spam/duplicates + a per-source detail string). Admin → صف آگهی‌ها shows a «تاریخچه جمع‌آوری» table of the last 15 runs (hover a row for the per-source breakdown), so admins can see how much each source found vs added over time. IngestionSummary gains TotalFetched. Migration: IngestionRuns table. Co-Authored-By: Claude Opus 4.8 --- src/JobsMedical.Web/Data/AppDbContext.cs | 1 + ...20260608024637_IngestionRunLog.Designer.cs | 1364 +++++++++++++++++ .../20260608024637_IngestionRunLog.cs | 43 + .../Migrations/AppDbContextModelSnapshot.cs | 38 + src/JobsMedical.Web/Models/IngestionRun.cs | 21 + src/JobsMedical.Web/Pages/Admin/Index.cshtml | 34 + .../Pages/Admin/Index.cshtml.cs | 2 + .../Services/Scraping/IngestionService.cs | 23 +- 8 files changed, 1525 insertions(+), 1 deletion(-) create mode 100644 src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.Designer.cs create mode 100644 src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.cs create mode 100644 src/JobsMedical.Web/Models/IngestionRun.cs diff --git a/src/JobsMedical.Web/Data/AppDbContext.cs b/src/JobsMedical.Web/Data/AppDbContext.cs index 550ed34..54254f7 100644 --- a/src/JobsMedical.Web/Data/AppDbContext.cs +++ b/src/JobsMedical.Web/Data/AppDbContext.cs @@ -30,6 +30,7 @@ public class AppDbContext : DbContext, IDataProtectionKeyContext public DbSet Reports => Set(); public DbSet FacilityDocuments => Set(); public DbSet JobAlerts => Set(); + public DbSet IngestionRuns => Set(); public DbSet Reviews => Set(); protected override void OnModelCreating(ModelBuilder b) diff --git a/src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.Designer.cs b/src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.Designer.cs new file mode 100644 index 0000000..87bbd6f --- /dev/null +++ b/src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.Designer.cs @@ -0,0 +1,1364 @@ +// +using System; +using JobsMedical.Web.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace JobsMedical.Web.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20260608024637_IngestionRunLog")] + partial class IngestionRunLog + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("JobsMedical.Web.Models.AppSetting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("AiApiKey") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("AiAutoApprove") + .HasColumnType("boolean"); + + b.Property("AiEnabled") + .HasColumnType("boolean"); + + b.Property("AiEndpoint") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("AiModel") + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("AiSystemPrompt") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.Property("AiUseProxy") + .HasColumnType("boolean"); + + b.Property("AutoIngestEnabled") + .HasColumnType("boolean"); + + b.Property("AutoPublishMinConfidence") + .HasColumnType("integer"); + + b.Property("BaleBotToken") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("BaleEnabled") + .HasColumnType("boolean"); + + b.Property("BaleUseProxy") + .HasColumnType("boolean"); + + b.Property("DemoMode") + .HasColumnType("boolean"); + + b.Property("DivarCity") + .HasMaxLength(60) + .HasColumnType("character varying(60)"); + + b.Property("DivarEnabled") + .HasColumnType("boolean"); + + b.Property("DivarQueries") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("DivarUseProxy") + .HasColumnType("boolean"); + + b.Property("IngestIntervalMinutes") + .HasColumnType("integer"); + + b.Property("IngestProxyEnabled") + .HasColumnType("boolean"); + + b.Property("IngestProxyUrl") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("MedjobsEnabled") + .HasColumnType("boolean"); + + b.Property("MedjobsMaxAds") + .HasColumnType("integer"); + + b.Property("MedjobsUseProxy") + .HasColumnType("boolean"); + + b.Property("Mode") + .HasColumnType("integer"); + + b.Property("NeshanMapKey") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("PushEnabled") + .HasColumnType("boolean"); + + b.Property("SmsApiKey") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("SmsEnabled") + .HasColumnType("boolean"); + + b.Property("SmsSender") + .HasMaxLength(30) + .HasColumnType("character varying(30)"); + + b.Property("SmsTemplate") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("TelegramChannels") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("TelegramEnabled") + .HasColumnType("boolean"); + + b.Property("TelegramUseProxy") + .HasColumnType("boolean"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("VapidPrivateKey") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("VapidPublicKey") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("VapidSubject") + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("WebNotificationsEnabled") + .HasColumnType("boolean"); + + b.Property("WebsiteUrls") + .HasMaxLength(4000) + .HasColumnType("character varying(4000)"); + + b.Property("WebsitesEnabled") + .HasColumnType("boolean"); + + b.Property("WebsitesUseProxy") + .HasColumnType("boolean"); + + b.HasKey("Id"); + + b.ToTable("AppSettings"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Application", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DoctorId") + .HasColumnType("integer"); + + b.Property("Message") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("ShiftId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("DoctorId"); + + b.HasIndex("ShiftId", "DoctorId") + .IsUnique(); + + b.ToTable("Applications"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.City", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("Province") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.HasKey("Id"); + + b.ToTable("Cities"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.District", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CityId") + .HasColumnType("integer"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.HasKey("Id"); + + b.HasIndex("CityId"); + + b.ToTable("Districts"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.DoctorProfile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Bio") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("CityId") + .HasColumnType("integer"); + + b.Property("IsVerified") + .HasColumnType("boolean"); + + b.Property("LicenseNo") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("RoleId") + .HasColumnType("integer"); + + b.Property("Specialty") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("UserId") + .HasColumnType("integer"); + + b.Property("YearsExperience") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CityId"); + + b.HasIndex("RoleId"); + + b.HasIndex("UserId") + .IsUnique(); + + b.ToTable("DoctorProfiles"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Facility", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Address") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("BaleId") + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("CityId") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DistrictId") + .HasColumnType("integer"); + + b.Property("IsDemo") + .HasColumnType("boolean"); + + b.Property("IsVerified") + .HasColumnType("boolean"); + + b.Property("Lat") + .HasColumnType("double precision"); + + b.Property("Lng") + .HasColumnType("double precision"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("OwnerUserId") + .HasColumnType("integer"); + + b.Property("Phone") + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Type") + .HasColumnType("integer"); + + b.Property("Verification") + .HasColumnType("integer"); + + b.Property("VerificationNote") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("VerificationRequestedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("CityId"); + + b.HasIndex("DistrictId"); + + b.HasIndex("OwnerUserId"); + + b.ToTable("Facilities"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.FacilityDocument", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("Data") + .IsRequired() + .HasColumnType("bytea"); + + b.Property("FacilityId") + .HasColumnType("integer"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("UploadedAt") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.HasIndex("FacilityId"); + + b.ToTable("FacilityDocuments"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.IngestionRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Detail") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("Duplicates") + .HasColumnType("integer"); + + b.Property("Fetched") + .HasColumnType("integer"); + + b.Property("Flagged") + .HasColumnType("integer"); + + b.Property("Published") + .HasColumnType("integer"); + + b.Property("Queued") + .HasColumnType("integer"); + + b.Property("RunAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Spam") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("IngestionRuns"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.InterestEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("EventType") + .HasColumnType("integer"); + + b.Property("JobOpeningId") + .HasColumnType("integer"); + + b.Property("ShiftId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("VisitorId") + .IsRequired() + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("JobOpeningId"); + + b.HasIndex("ShiftId"); + + b.HasIndex("VisitorId", "CreatedAt"); + + b.ToTable("InterestEvents"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.JobAlert", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CityId") + .HasColumnType("integer"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("DistrictId") + .HasColumnType("integer"); + + b.Property("EmploymentType") + .HasColumnType("integer"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("Label") + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("MinPay") + .HasColumnType("bigint"); + + b.Property("RoleId") + .HasColumnType("integer"); + + b.Property("Scope") + .HasColumnType("integer"); + + b.Property("ShiftType") + .HasColumnType("integer"); + + b.Property("UserId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("CityId"); + + b.HasIndex("IsActive"); + + b.HasIndex("RoleId"); + + b.HasIndex("UserId"); + + b.ToTable("JobAlerts"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.JobOpening", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("EmploymentType") + .HasColumnType("integer"); + + b.Property("FacilityId") + .HasColumnType("integer"); + + b.Property("GenderRequirement") + .HasColumnType("integer"); + + b.Property("Requirements") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("RoleId") + .HasColumnType("integer"); + + b.Property("SalaryMax") + .HasColumnType("bigint"); + + b.Property("SalaryMin") + .HasColumnType("bigint"); + + b.Property("Source") + .HasColumnType("integer"); + + b.Property("SourceUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.HasKey("Id"); + + b.HasIndex("FacilityId"); + + b.HasIndex("RoleId"); + + b.HasIndex("Status"); + + b.ToTable("JobOpenings"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Notification", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Body") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("IsRead") + .HasColumnType("boolean"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Url") + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.Property("UserId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("UserId", "IsRead", "CreatedAt"); + + b.ToTable("Notifications"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.RawListing", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Confidence") + .HasColumnType("integer"); + + b.Property("ContentHash") + .HasMaxLength(64) + .HasColumnType("character varying(64)"); + + b.Property("FetchedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LinkedShiftId") + .HasColumnType("integer"); + + b.Property("ParsedJson") + .HasColumnType("text"); + + b.Property("RawText") + .IsRequired() + .HasColumnType("text"); + + b.Property("SourceChannel") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("SourceUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("ValidationNotes") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.HasKey("Id"); + + b.HasIndex("ContentHash"); + + b.HasIndex("LinkedShiftId"); + + b.HasIndex("Status"); + + b.ToTable("RawListings"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Report", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Reason") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("ReporterUserId") + .HasColumnType("integer"); + + b.Property("ReporterVisitorId") + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("TargetId") + .HasColumnType("integer"); + + b.Property("TargetLabel") + .HasMaxLength(160) + .HasColumnType("character varying(160)"); + + b.Property("TargetType") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Status"); + + b.ToTable("Reports"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Review", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Comment") + .HasMaxLength(1000) + .HasColumnType("character varying(1000)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FacilityId") + .HasColumnType("integer"); + + b.Property("IsApproved") + .HasColumnType("boolean"); + + b.Property("Stars") + .HasColumnType("integer"); + + b.Property("UserId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.HasIndex("FacilityId", "UserId") + .IsUnique(); + + b.ToTable("Reviews"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Category") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("character varying(50)"); + + b.Property("IsActive") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("SortOrder") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Roles"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Shift", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Date") + .HasColumnType("date"); + + b.Property("Description") + .HasMaxLength(1500) + .HasColumnType("character varying(1500)"); + + b.Property("EndTime") + .HasColumnType("time without time zone"); + + b.Property("FacilityId") + .HasColumnType("integer"); + + b.Property("GenderRequirement") + .HasColumnType("integer"); + + b.Property("PayAmount") + .HasColumnType("bigint"); + + b.Property("PayType") + .HasColumnType("integer"); + + b.Property("RoleId") + .HasColumnType("integer"); + + b.Property("SharePercent") + .HasColumnType("integer"); + + b.Property("ShiftType") + .HasColumnType("integer"); + + b.Property("Source") + .HasColumnType("integer"); + + b.Property("SourceUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)"); + + b.Property("SpecialtyRequired") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("StartTime") + .HasColumnType("time without time zone"); + + b.Property("Status") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("FacilityId"); + + b.HasIndex("RoleId"); + + b.HasIndex("Date", "Status"); + + b.ToTable("Shifts"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.User", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Avatar") + .HasColumnType("bytea"); + + b.Property("AvatarContentType") + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("BanReason") + .HasMaxLength(300) + .HasColumnType("character varying(300)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FullName") + .HasMaxLength(150) + .HasColumnType("character varying(150)"); + + b.Property("IsBanned") + .HasColumnType("boolean"); + + b.Property("IsPhoneVerified") + .HasColumnType("boolean"); + + b.Property("Phone") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("character varying(20)"); + + b.Property("Resume") + .HasColumnType("bytea"); + + b.Property("ResumeContentType") + .HasMaxLength(120) + .HasColumnType("character varying(120)"); + + b.Property("ResumeFileName") + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("Role") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Phone") + .IsUnique(); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.UserPreferences", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CityId") + .HasColumnType("integer"); + + b.Property("Gender") + .HasColumnType("integer"); + + b.Property("MinPay") + .HasColumnType("bigint"); + + b.Property("PreferredShiftType") + .HasColumnType("integer"); + + b.Property("RoleId") + .HasColumnType("integer"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("VisitorId") + .IsRequired() + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("CityId"); + + b.HasIndex("RoleId"); + + b.HasIndex("VisitorId") + .IsUnique(); + + b.ToTable("UserPreferences"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Visitor", b => + { + b.Property("Id") + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("LastSeenAt") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("Visitors"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.WebPushSubscription", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Auth") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Endpoint") + .IsRequired() + .HasMaxLength(600) + .HasColumnType("character varying(600)"); + + b.Property("P256dh") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)"); + + b.Property("VisitorId") + .HasMaxLength(36) + .HasColumnType("character varying(36)"); + + b.HasKey("Id"); + + b.HasIndex("Endpoint") + .IsUnique(); + + b.ToTable("WebPushSubscriptions"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("FriendlyName") + .HasColumnType("text"); + + b.Property("Xml") + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Application", b => + { + b.HasOne("JobsMedical.Web.Models.User", "Doctor") + .WithMany("Applications") + .HasForeignKey("DoctorId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("JobsMedical.Web.Models.Shift", "Shift") + .WithMany("Applications") + .HasForeignKey("ShiftId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Doctor"); + + b.Navigation("Shift"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.District", b => + { + b.HasOne("JobsMedical.Web.Models.City", "City") + .WithMany() + .HasForeignKey("CityId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("City"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.DoctorProfile", b => + { + b.HasOne("JobsMedical.Web.Models.City", "City") + .WithMany() + .HasForeignKey("CityId"); + + b.HasOne("JobsMedical.Web.Models.Role", "Role") + .WithMany() + .HasForeignKey("RoleId"); + + b.HasOne("JobsMedical.Web.Models.User", "User") + .WithOne("DoctorProfile") + .HasForeignKey("JobsMedical.Web.Models.DoctorProfile", "UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("City"); + + b.Navigation("Role"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Facility", b => + { + b.HasOne("JobsMedical.Web.Models.City", "City") + .WithMany("Facilities") + .HasForeignKey("CityId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("JobsMedical.Web.Models.District", "District") + .WithMany("Facilities") + .HasForeignKey("DistrictId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("JobsMedical.Web.Models.User", "OwnerUser") + .WithMany() + .HasForeignKey("OwnerUserId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("City"); + + b.Navigation("District"); + + b.Navigation("OwnerUser"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.FacilityDocument", b => + { + b.HasOne("JobsMedical.Web.Models.Facility", "Facility") + .WithMany("Documents") + .HasForeignKey("FacilityId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Facility"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.InterestEvent", b => + { + b.HasOne("JobsMedical.Web.Models.JobOpening", "JobOpening") + .WithMany() + .HasForeignKey("JobOpeningId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("JobsMedical.Web.Models.Shift", "Shift") + .WithMany() + .HasForeignKey("ShiftId") + .OnDelete(DeleteBehavior.Cascade); + + b.HasOne("JobsMedical.Web.Models.Visitor", "Visitor") + .WithMany("Events") + .HasForeignKey("VisitorId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("JobOpening"); + + b.Navigation("Shift"); + + b.Navigation("Visitor"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.JobAlert", b => + { + b.HasOne("JobsMedical.Web.Models.City", "City") + .WithMany() + .HasForeignKey("CityId"); + + b.HasOne("JobsMedical.Web.Models.Role", "Role") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("JobsMedical.Web.Models.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("City"); + + b.Navigation("Role"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.JobOpening", b => + { + b.HasOne("JobsMedical.Web.Models.Facility", "Facility") + .WithMany() + .HasForeignKey("FacilityId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("JobsMedical.Web.Models.Role", "Role") + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Facility"); + + b.Navigation("Role"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Notification", b => + { + b.HasOne("JobsMedical.Web.Models.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("User"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.RawListing", b => + { + b.HasOne("JobsMedical.Web.Models.Shift", "LinkedShift") + .WithMany() + .HasForeignKey("LinkedShiftId"); + + b.Navigation("LinkedShift"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Review", b => + { + b.HasOne("JobsMedical.Web.Models.Facility", "Facility") + .WithMany() + .HasForeignKey("FacilityId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("JobsMedical.Web.Models.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Facility"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Shift", b => + { + b.HasOne("JobsMedical.Web.Models.Facility", "Facility") + .WithMany("Shifts") + .HasForeignKey("FacilityId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("JobsMedical.Web.Models.Role", "Role") + .WithMany("Shifts") + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Facility"); + + b.Navigation("Role"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.UserPreferences", b => + { + b.HasOne("JobsMedical.Web.Models.City", "City") + .WithMany() + .HasForeignKey("CityId"); + + b.HasOne("JobsMedical.Web.Models.Role", "Role") + .WithMany() + .HasForeignKey("RoleId"); + + b.HasOne("JobsMedical.Web.Models.Visitor", "Visitor") + .WithOne("Preferences") + .HasForeignKey("JobsMedical.Web.Models.UserPreferences", "VisitorId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("City"); + + b.Navigation("Role"); + + b.Navigation("Visitor"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Visitor", b => + { + b.HasOne("JobsMedical.Web.Models.User", "User") + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.SetNull); + + b.Navigation("User"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.City", b => + { + b.Navigation("Facilities"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.District", b => + { + b.Navigation("Facilities"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Facility", b => + { + b.Navigation("Documents"); + + b.Navigation("Shifts"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Role", b => + { + b.Navigation("Shifts"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Shift", b => + { + b.Navigation("Applications"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.User", b => + { + b.Navigation("Applications"); + + b.Navigation("DoctorProfile"); + }); + + modelBuilder.Entity("JobsMedical.Web.Models.Visitor", b => + { + b.Navigation("Events"); + + b.Navigation("Preferences"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.cs b/src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.cs new file mode 100644 index 0000000..c8ba344 --- /dev/null +++ b/src/JobsMedical.Web/Migrations/20260608024637_IngestionRunLog.cs @@ -0,0 +1,43 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace JobsMedical.Web.Migrations +{ + /// + public partial class IngestionRunLog : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "IngestionRuns", + columns: table => new + { + Id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + RunAt = table.Column(type: "timestamp with time zone", nullable: false), + Fetched = table.Column(type: "integer", nullable: false), + Queued = table.Column(type: "integer", nullable: false), + Published = table.Column(type: "integer", nullable: false), + Flagged = table.Column(type: "integer", nullable: false), + Spam = table.Column(type: "integer", nullable: false), + Duplicates = table.Column(type: "integer", nullable: false), + Detail = table.Column(type: "character varying(2000)", maxLength: 2000, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_IngestionRuns", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "IngestionRuns"); + } + } +} diff --git a/src/JobsMedical.Web/Migrations/AppDbContextModelSnapshot.cs b/src/JobsMedical.Web/Migrations/AppDbContextModelSnapshot.cs index d5735c5..d3cf14f 100644 --- a/src/JobsMedical.Web/Migrations/AppDbContextModelSnapshot.cs +++ b/src/JobsMedical.Web/Migrations/AppDbContextModelSnapshot.cs @@ -421,6 +421,44 @@ namespace JobsMedical.Web.Migrations b.ToTable("FacilityDocuments"); }); + modelBuilder.Entity("JobsMedical.Web.Models.IngestionRun", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Detail") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)"); + + b.Property("Duplicates") + .HasColumnType("integer"); + + b.Property("Fetched") + .HasColumnType("integer"); + + b.Property("Flagged") + .HasColumnType("integer"); + + b.Property("Published") + .HasColumnType("integer"); + + b.Property("Queued") + .HasColumnType("integer"); + + b.Property("RunAt") + .HasColumnType("timestamp with time zone"); + + b.Property("Spam") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("IngestionRuns"); + }); + modelBuilder.Entity("JobsMedical.Web.Models.InterestEvent", b => { b.Property("Id") diff --git a/src/JobsMedical.Web/Models/IngestionRun.cs b/src/JobsMedical.Web/Models/IngestionRun.cs new file mode 100644 index 0000000..8261fad --- /dev/null +++ b/src/JobsMedical.Web/Models/IngestionRun.cs @@ -0,0 +1,21 @@ +using System.ComponentModel.DataAnnotations; + +namespace JobsMedical.Web.Models; + +/// One ingestion run's outcome — kept so admins see a history of what was crawled, +/// how much was found, queued, published, flagged, etc. (with a per-source breakdown). +public class IngestionRun +{ + public int Id { get; set; } + public DateTime RunAt { get; set; } = DateTime.UtcNow; + + public int Fetched { get; set; } // total items pulled from all sources + public int Queued { get; set; } // sent to the review queue + public int Published { get; set; } // auto-published + public int Flagged { get; set; } // needs-review + public int Spam { get; set; } // discarded as spam/irrelevant + public int Duplicates { get; set; } // skipped (already seen) + + /// Human-readable per-source breakdown, e.g. "دیوار: یافت ۱۲…؛ مدجابز: یافت ۴۰…". + [MaxLength(2000)] public string? Detail { get; set; } +} diff --git a/src/JobsMedical.Web/Pages/Admin/Index.cshtml b/src/JobsMedical.Web/Pages/Admin/Index.cshtml index 01e7d31..60beb26 100644 --- a/src/JobsMedical.Web/Pages/Admin/Index.cshtml +++ b/src/JobsMedical.Web/Pages/Admin/Index.cshtml @@ -62,6 +62,40 @@
+ @if (Model.Runs.Count > 0) + { +

تاریخچه جمع‌آوری

+
+ + + + + + + + + + + + + + @foreach (var run in Model.Runs) + { + + + + + + + + + + } + +
زمانیافت‌شدهصفمنتشرپرچماسپمتکراری
@JalaliDate.ToLongDate(DateOnly.FromDateTime(run.RunAt)) @run.RunAt.ToString("HH:mm")@JalaliDate.ToPersianDigits(run.Fetched.ToString())@JalaliDate.ToPersianDigits(run.Queued.ToString())@JalaliDate.ToPersianDigits(run.Published.ToString())@JalaliDate.ToPersianDigits(run.Flagged.ToString())@JalaliDate.ToPersianDigits(run.Spam.ToString())@JalaliDate.ToPersianDigits(run.Duplicates.ToString())
+

جزئیات هر منبع را با نگه‌داشتن نشانگر روی هر ردیف ببین. لاگ کامل: docker logs hamkadr_api

+
+ }

صف بررسی

@if (Model.Queue.Count == 0) { diff --git a/src/JobsMedical.Web/Pages/Admin/Index.cshtml.cs b/src/JobsMedical.Web/Pages/Admin/Index.cshtml.cs index ccf49ea..21f3036 100644 --- a/src/JobsMedical.Web/Pages/Admin/Index.cshtml.cs +++ b/src/JobsMedical.Web/Pages/Admin/Index.cshtml.cs @@ -25,6 +25,7 @@ public class IndexModel : PageModel public IReadOnlyList SourceNames { get; private set; } = new List(); public int PublishedShifts { get; private set; } public int PublishedJobs { get; private set; } + public List Runs { get; private set; } = new(); [BindProperty] public string? SourceChannel { get; set; } [BindProperty] public string? RawText { get; set; } @@ -67,5 +68,6 @@ public class IndexModel : PageModel SourceNames = _ingest.SourceNames; PublishedShifts = await _db.Shifts.CountAsync(s => s.Source != ShiftSource.Direct); PublishedJobs = await _db.JobOpenings.CountAsync(); + Runs = await _db.IngestionRuns.OrderByDescending(r => r.RunAt).Take(15).ToListAsync(); } } diff --git a/src/JobsMedical.Web/Services/Scraping/IngestionService.cs b/src/JobsMedical.Web/Services/Scraping/IngestionService.cs index 857d08b..09bd795 100644 --- a/src/JobsMedical.Web/Services/Scraping/IngestionService.cs +++ b/src/JobsMedical.Web/Services/Scraping/IngestionService.cs @@ -11,6 +11,7 @@ public record SourceResult(string Source, int Fetched, int Queued, int Published public record IngestionSummary(List Sources) { + public int TotalFetched => Sources.Sum(s => s.Fetched); public int TotalQueued => Sources.Sum(s => s.Queued); public int TotalPublished => Sources.Sum(s => s.Published); public int TotalFlagged => Sources.Sum(s => s.Flagged); @@ -108,7 +109,27 @@ public class IngestionService source.Name, fetched, queued, published, flagged, spam, dupes); } - return new IngestionSummary(results); + var summary = new IngestionSummary(results); + + // Persist a run-log row so admins get a crawl history (with a per-source breakdown). + if (results.Count > 0) + { + var detail = string.Join("؛ ", results.Select(r => + $"{r.Source}: یافت {r.Fetched}، صف {r.Queued}، منتشر {r.Published}، پرچم {r.Flagged}، اسپم {r.Spam}، تکراری {r.Duplicates}")); + _db.IngestionRuns.Add(new IngestionRun + { + Fetched = summary.TotalFetched, + Queued = summary.TotalQueued, + Published = summary.TotalPublished, + Flagged = summary.TotalFlagged, + Spam = summary.TotalSpam, + Duplicates = summary.TotalDuplicates, + Detail = detail.Length > 2000 ? detail[..2000] : detail, + }); + await _db.SaveChangesAsync(ct); + } + + return summary; } private static (RawListingStatus status, string? reason, int confidence) Decide(