[Ingest] Per-source proxy toggle instead of one global switch
Each ingestion source now decides independently whether to route through the proxy: added TelegramUseProxy/BaleUseProxy/DivarUseProxy/MedjobsUseProxy/WebsitesUseProxy flags (migration). ScrapeHttpClients.For(s, useProxy) takes the source's own flag; a source is proxied only when its flag is on AND a proxy URL is set. Settings 'sources' tab: removed the global enable checkbox, kept the proxy address field, and added an «از پروکسی استفاده شود» checkbox under each source. Old IngestProxyEnabled column kept for compatibility but no longer gates routing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -27,7 +27,7 @@ public class BaleListingSource : IListingSource
|
||||
|
||||
try
|
||||
{
|
||||
var client = _clients.For(s);
|
||||
var client = _clients.For(s, s.BaleUseProxy);
|
||||
var body = await client.GetStringAsync($"{BaseUrl}/bot{s.BaleBotToken}/getUpdates", ct);
|
||||
using var doc = JsonDocument.Parse(body);
|
||||
if (!doc.RootElement.TryGetProperty("result", out var result) || result.ValueKind != JsonValueKind.Array)
|
||||
|
||||
@@ -29,7 +29,7 @@ public class DivarListingSource : IListingSource
|
||||
if (!s.DivarEnabled || queries.Count == 0) return Array.Empty<ScrapedItem>();
|
||||
var city = string.IsNullOrWhiteSpace(s.DivarCity) ? "tehran" : s.DivarCity.Trim();
|
||||
|
||||
var client = _clients.For(s);
|
||||
var client = _clients.For(s, s.DivarUseProxy);
|
||||
var items = new List<ScrapedItem>();
|
||||
foreach (var q in queries)
|
||||
{
|
||||
|
||||
@@ -28,7 +28,7 @@ public class MedjobsListingSource : IListingSource
|
||||
{
|
||||
if (!s.MedjobsEnabled) return Array.Empty<ScrapedItem>();
|
||||
var max = Math.Clamp(s.MedjobsMaxAds, 1, 500);
|
||||
var client = _clients.For(s);
|
||||
var client = _clients.For(s, s.MedjobsUseProxy);
|
||||
|
||||
try
|
||||
{
|
||||
|
||||
@@ -19,10 +19,11 @@ public sealed class ScrapeHttpClients : IDisposable
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, HttpClient> _cache = new();
|
||||
|
||||
/// <summary>The HttpClient for the given settings — proxied when enabled, direct otherwise.</summary>
|
||||
public HttpClient For(AppSetting s)
|
||||
/// <summary>The HttpClient for a source — proxied only when that source opts in AND a proxy
|
||||
/// URL is configured; otherwise a direct client. Pass the source's own per-source flag.</summary>
|
||||
public HttpClient For(AppSetting s, bool useProxy)
|
||||
{
|
||||
var key = (s.IngestProxyEnabled && !string.IsNullOrWhiteSpace(s.IngestProxyUrl))
|
||||
var key = (useProxy && !string.IsNullOrWhiteSpace(s.IngestProxyUrl))
|
||||
? s.IngestProxyUrl.Trim()
|
||||
: "direct";
|
||||
|
||||
|
||||
@@ -44,8 +44,12 @@ public class SettingsService
|
||||
s.DemoMode = incoming.DemoMode;
|
||||
s.WebsitesEnabled = incoming.WebsitesEnabled;
|
||||
s.WebsiteUrls = incoming.WebsiteUrls?.Trim();
|
||||
s.IngestProxyEnabled = incoming.IngestProxyEnabled;
|
||||
s.IngestProxyUrl = incoming.IngestProxyUrl?.Trim();
|
||||
s.TelegramUseProxy = incoming.TelegramUseProxy;
|
||||
s.BaleUseProxy = incoming.BaleUseProxy;
|
||||
s.DivarUseProxy = incoming.DivarUseProxy;
|
||||
s.MedjobsUseProxy = incoming.MedjobsUseProxy;
|
||||
s.WebsitesUseProxy = incoming.WebsitesUseProxy;
|
||||
s.DivarEnabled = incoming.DivarEnabled;
|
||||
s.DivarCity = string.IsNullOrWhiteSpace(incoming.DivarCity) ? "tehran" : incoming.DivarCity.Trim();
|
||||
s.DivarQueries = incoming.DivarQueries?.Trim();
|
||||
|
||||
@@ -26,7 +26,7 @@ public class TelegramListingSource : IListingSource
|
||||
var channels = AppSetting.SplitList(s.TelegramChannels);
|
||||
if (!s.TelegramEnabled || channels.Count == 0) return Array.Empty<ScrapedItem>();
|
||||
|
||||
var client = _clients.For(s);
|
||||
var client = _clients.For(s, s.TelegramUseProxy);
|
||||
var items = new List<ScrapedItem>();
|
||||
foreach (var ch in channels.Select(c => c.TrimStart('@')).Where(c => c.Length > 0))
|
||||
{
|
||||
|
||||
@@ -27,7 +27,7 @@ public class WebsiteListingSource : IListingSource
|
||||
var urls = AppSetting.SplitList(s.WebsiteUrls);
|
||||
if (!s.WebsitesEnabled || urls.Count == 0) return Array.Empty<ScrapedItem>();
|
||||
|
||||
var client = _clients.For(s);
|
||||
var client = _clients.For(s, s.WebsitesUseProxy);
|
||||
var items = new List<ScrapedItem>();
|
||||
foreach (var url in urls.Where(u => u.StartsWith("http")))
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user