refactor: implement worker pool pattern, add persistent task queue, and decouple target loading logic
This commit is contained in:
+58
-19
@@ -1,7 +1,6 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -9,20 +8,20 @@ import (
|
||||
|
||||
"github.com/hireme/hireme/internal/config"
|
||||
"github.com/hireme/hireme/internal/db"
|
||||
"github.com/hireme/hireme/internal/service"
|
||||
"github.com/hireme/hireme/internal/worker"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
cfg *config.Config
|
||||
db *db.DB
|
||||
pipeline *service.PipelineService
|
||||
cfg *config.Config
|
||||
db *db.DB
|
||||
pool *worker.WorkerPool
|
||||
}
|
||||
|
||||
func NewServer(cfg *config.Config, database *db.DB, pipeline *service.PipelineService) *Server {
|
||||
func NewServer(cfg *config.Config, database *db.DB, pool *worker.WorkerPool) *Server {
|
||||
return &Server{
|
||||
cfg: cfg,
|
||||
db: database,
|
||||
pipeline: pipeline,
|
||||
cfg: cfg,
|
||||
db: database,
|
||||
pool: pool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +35,9 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /jobs/{id}/analyze", s.handleAnalyzeJob)
|
||||
mux.HandleFunc("POST /applications", s.handleCreateApplication)
|
||||
mux.HandleFunc("PATCH /applications/{id}/status", s.handleUpdateAppStatus)
|
||||
mux.HandleFunc("POST /pipeline/run", s.handleRunPipeline)
|
||||
mux.HandleFunc("POST /api/tasks/discover", s.handleDiscoverTasks)
|
||||
mux.HandleFunc("POST /api/tasks/clean", s.handleCleanTasks)
|
||||
mux.HandleFunc("POST /api/tasks/toggle", s.handleToggleTasks)
|
||||
}
|
||||
|
||||
func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -175,14 +176,16 @@ func (s *Server) handleAnalyzeJob(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
job, err := s.pipeline.AnalyzeJobByID(r.Context(), id)
|
||||
// Enqueue analyze task instead of synchronous processing
|
||||
payload := worker.AnalyzePayload{RawJobID: int64(id)}
|
||||
err = s.db.EnqueueTask("ANALYZE", payload)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
http.Error(w, "Failed to enqueue analysis task: "+err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_ = RenderJobDetail(w, job)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_, _ = w.Write([]byte(`<span class="text-xs px-2 py-1 bg-yellow-100 text-yellow-800 rounded">Analysis Queued</span>`))
|
||||
}
|
||||
|
||||
func (s *Server) handleCreateApplication(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -269,12 +272,48 @@ func (s *Server) handleUpdateAppStatus(w http.ResponseWriter, r *http.Request) {
|
||||
_ = RenderKanbanBoard(w, updatedApps)
|
||||
}
|
||||
|
||||
func (s *Server) handleRunPipeline(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := context.Background()
|
||||
_, _, err := s.pipeline.RunPipeline(ctx)
|
||||
func (s *Server) handleDiscoverTasks(w http.ResponseWriter, r *http.Request) {
|
||||
targets, err := s.db.GetAllTargets()
|
||||
if err != nil {
|
||||
log.Printf("[Server] Pipeline run error: %v", err)
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
s.handleGetJobs(w, r)
|
||||
count := 0
|
||||
for _, target := range targets {
|
||||
if !target.IsActive {
|
||||
continue
|
||||
}
|
||||
payload := worker.DiscoverPayload{
|
||||
TargetID: target.ID,
|
||||
TargetURL: target.TargetURL,
|
||||
Platform: target.Platform,
|
||||
}
|
||||
if err := s.db.EnqueueTask("DISCOVER", payload); err == nil {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status": "success", "message": "Queued ` + strconv.Itoa(count) + ` discovery tasks"}`))
|
||||
}
|
||||
|
||||
func (s *Server) handleCleanTasks(w http.ResponseWriter, r *http.Request) {
|
||||
err := s.db.ClearTaskQueue()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status": "success"}`))
|
||||
}
|
||||
|
||||
func (s *Server) handleToggleTasks(w http.ResponseWriter, r *http.Request) {
|
||||
isRunning := s.pool.Toggle()
|
||||
status := "paused"
|
||||
if isRunning {
|
||||
status = "running"
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"status": "` + status + `"}`))
|
||||
}
|
||||
|
||||
+30
-14
@@ -180,16 +180,26 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
|
||||
<span class="font-semibold text-indigo-300 uppercase">{{ .LLMProvider }}</span>
|
||||
</div>
|
||||
|
||||
<button hx-post="/pipeline/run"
|
||||
hx-target="#jobs-container"
|
||||
hx-indicator="#run-spinner"
|
||||
class="relative inline-flex items-center justify-center px-4 py-2 text-xs font-semibold text-white transition-all bg-indigo-600 rounded-lg shadow-md hover:bg-indigo-500 active:scale-95 shadow-indigo-500/25">
|
||||
<svg id="run-spinner" class="htmx-indicator animate-spin -ml-1 mr-2 h-4 w-4 text-white" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
|
||||
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
|
||||
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
|
||||
</svg>
|
||||
<span>⚡ Run Pipeline</span>
|
||||
</button>
|
||||
<div class="flex items-center space-x-2">
|
||||
<button hx-post="/api/tasks/discover"
|
||||
hx-swap="none"
|
||||
hx-on::after-request="location.reload()"
|
||||
class="relative inline-flex items-center justify-center px-3 py-1.5 text-xs font-semibold text-white transition-all bg-indigo-600 rounded-lg shadow-md hover:bg-indigo-500 active:scale-95 shadow-indigo-500/25">
|
||||
<span>🔍 Discover Jobs</span>
|
||||
</button>
|
||||
<button hx-post="/api/tasks/clean"
|
||||
hx-swap="none"
|
||||
hx-on::after-request="location.reload()"
|
||||
class="relative inline-flex items-center justify-center px-3 py-1.5 text-xs font-semibold text-white transition-all bg-red-600 rounded-lg shadow-md hover:bg-red-500 active:scale-95 shadow-red-500/25">
|
||||
<span>🗑️ Clean Queue</span>
|
||||
</button>
|
||||
<button hx-post="/api/tasks/toggle"
|
||||
hx-swap="none"
|
||||
hx-on::after-request="location.reload()"
|
||||
class="relative inline-flex items-center justify-center px-3 py-1.5 text-xs font-semibold text-white transition-all bg-emerald-600 rounded-lg shadow-md hover:bg-emerald-500 active:scale-95 shadow-emerald-500/25">
|
||||
<span>⚙️ Toggle Workers</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
@@ -454,9 +464,15 @@ const jobListHTML = `{{ define "jobs" }}
|
||||
★ {{ printf "%.0f" .Analysis.Score }} Qualified
|
||||
</span>
|
||||
{{ else }}
|
||||
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-[10px] font-semibold bg-rose-500/10 text-rose-400 border border-rose-500/20">
|
||||
Not Qualified
|
||||
</span>
|
||||
<div class="flex flex-col gap-1 items-start">
|
||||
<span class="inline-flex items-center px-2.5 py-0.5 rounded-full text-[10px] font-semibold bg-rose-500/10 text-rose-400 border border-rose-500/20">
|
||||
Not Qualified
|
||||
</span>
|
||||
{{ $missing := parseJSON .Analysis.MissingHighlights }}
|
||||
{{ if $missing }}
|
||||
<span class="text-[9px] text-rose-400/80 line-clamp-2 max-w-[150px] leading-tight" title="{{ index $missing 0 }}">{{ index $missing 0 }}</span>
|
||||
{{ end }}
|
||||
</div>
|
||||
{{ end }}
|
||||
{{ else }}
|
||||
<span class="text-slate-500 text-[11px]">-</span>
|
||||
@@ -504,7 +520,7 @@ const jobListHTML = `{{ define "jobs" }}
|
||||
{{ else }}
|
||||
<tr>
|
||||
<td colspan="7" class="px-4 py-8 text-center text-slate-500">
|
||||
No job postings found yet. Click <strong>⚡ Run Pipeline</strong> to ingest jobs.
|
||||
No job postings found yet. Click <strong>🔍 Discover Jobs</strong> to enqueue crawler tasks.
|
||||
</td>
|
||||
</tr>
|
||||
{{ end }}
|
||||
|
||||
Reference in New Issue
Block a user