diff options
| -rw-r--r-- | conf/general.yml-example | 8 | ||||
| -rw-r--r-- | perllib/Open311/GetServiceRequestUpdates.pm | 22 |
2 files changed, 26 insertions, 4 deletions
diff --git a/conf/general.yml-example b/conf/general.yml-example index d7078a217..243d077f0 100644 --- a/conf/general.yml-example +++ b/conf/general.yml-example @@ -262,5 +262,9 @@ LOGIN_REQUIRED: 0 # NB: This also disables all Facebook/Twitter logins. SIGNUPS_DISABLED: 0 -# Setting this variable to more than 1 will let fetch-comments run in parallel -FETCH_COMMENTS_PROCESSES: 0 +# Setting these variable to more than 1 will let fetch-comments run in parallel +# with MIN to MAX children (new children will be added if a child takes longer +# than TIMEOUT on one body). +FETCH_COMMENTS_PROCESSES_MIN: 0 +FETCH_COMMENTS_PROCESSES_MAX: 0 +FETCH_COMMENTS_PROCESS_TIMEOUT: 0 diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm index e27a08068..09b1f6b26 100644 --- a/perllib/Open311/GetServiceRequestUpdates.pm +++ b/perllib/Open311/GetServiceRequestUpdates.pm @@ -38,8 +38,26 @@ sub fetch { $bodies = $bodies->search( { name => $self->body } ); } - my $procs = FixMyStreet->config('FETCH_COMMENTS_PROCESSES') || 0; - my $pm = Parallel::ForkManager->new(FixMyStreet->test_mode ? 0 : $procs); + my $procs_min = FixMyStreet->config('FETCH_COMMENTS_PROCESSES_MIN') || 0; + my $procs_max = FixMyStreet->config('FETCH_COMMENTS_PROCESSES_MAX'); + my $procs_timeout = FixMyStreet->config('FETCH_COMMENTS_PROCESS_TIMEOUT'); + + my $pm = Parallel::ForkManager->new(FixMyStreet->test_mode ? 0 : $procs_min); + + if ($procs_max && $procs_timeout) { + my %workers; + $pm->run_on_wait(sub { + while (my ($pid, $started_at) = each %workers) { + next unless time() - $started_at > $procs_timeout; + next if $pm->max_procs == $procs_max; + $pm->set_max_procs($pm->max_procs + 1); + delete $workers{$pid}; # Only want to increase once per long-running thing + } + }, 1); + $pm->run_on_start(sub { my $pid = shift; $workers{$pid} = time(); }); + $pm->run_on_finish(sub { my $pid = shift; delete $workers{$pid}; }); + } + while ( my $body = $bodies->next ) { $pm->start and next; |