diff options
| author | Matthew Somerville <matthew@mysociety.org> | 2020-03-10 17:09:55 +0000 |
|---|---|---|
| committer | Matthew Somerville <matthew@mysociety.org> | 2020-04-02 17:14:21 +0100 |
| commit | 46300feaf7481ef4dac67ee312becf856c184135 (patch) | |
| tree | 7e9fd914ca30bc47f9ed2309c37ed143f56e768c | |
| parent | 1f0473c9844b6e32338d1c1af6e368f3387e8890 (diff) | |
Parallelize fetching updates.
Allow a configurable number of bodies to fetch updates simultaneously.
| -rw-r--r-- | conf/general.yml-example | 3 | ||||
| -rw-r--r-- | cpanfile | 1 | ||||
| -rw-r--r-- | cpanfile.snapshot | 18 | ||||
| -rw-r--r-- | perllib/FixMyStreet.pm | 1 | ||||
| -rw-r--r-- | perllib/Open311/GetServiceRequestUpdates.pm | 9 |
5 files changed, 32 insertions, 0 deletions
diff --git a/conf/general.yml-example b/conf/general.yml-example index 8e8d0f76a..d7078a217 100644 --- a/conf/general.yml-example +++ b/conf/general.yml-example @@ -261,3 +261,6 @@ LOGIN_REQUIRED: 0 # If you want to stop new users from registering, set this to 1. # NB: This also disables all Facebook/Twitter logins. SIGNUPS_DISABLED: 0 + +# Setting this variable to more than 1 will let fetch-comments run in parallel +FETCH_COMMENTS_PROCESSES: 0 @@ -110,6 +110,7 @@ requires 'Net::OAuth'; requires 'Net::Twitter::Lite::WithAPIv1_1', '0.12008'; requires 'Number::Phone', '3.5000'; requires 'OIDC::Lite'; +requires 'Parallel::ForkManager'; requires 'Path::Class'; requires 'POSIX'; requires 'Readonly'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 42c125da6..8c55fec0c 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -6028,6 +6028,24 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.008001 + Parallel-ForkManager-2.02 + pathname: Y/YA/YANICK/Parallel-ForkManager-2.02.tar.gz + provides: + Parallel::ForkManager 2.02 + Parallel::ForkManager::Child 2.02 + requirements: + Carp 0 + ExtUtils::MakeMaker 0 + File::Path 0 + File::Spec 0 + File::Temp 0 + Moo 0 + Moo::Role 0 + POSIX 0 + Storable 0 + perl 5.006 + strict 0 + warnings 0 Params-Classify-0.015 pathname: Z/ZE/ZEFRAM/Params-Classify-0.015.tar.gz provides: diff --git a/perllib/FixMyStreet.pm b/perllib/FixMyStreet.pm index f6a69928b..94ba7685f 100644 --- a/perllib/FixMyStreet.pm +++ b/perllib/FixMyStreet.pm @@ -154,6 +154,7 @@ sub dbic_connect_info { my $dbi_args = { AutoCommit => 1, + AutoInactiveDestroy => 1, }; my $local_time_zone = local_time_zone(); my $dbic_args = { diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm index a407d8e37..e27a08068 100644 --- a/perllib/Open311/GetServiceRequestUpdates.pm +++ b/perllib/Open311/GetServiceRequestUpdates.pm @@ -2,6 +2,7 @@ package Open311::GetServiceRequestUpdates; use Moo; use Open311; +use Parallel::ForkManager; use FixMyStreet::DB; use FixMyStreet::App::Model::PhotoSet; use DateTime::Format::W3CDTF; @@ -37,7 +38,11 @@ sub fetch { $bodies = $bodies->search( { name => $self->body } ); } + my $procs = FixMyStreet->config('FETCH_COMMENTS_PROCESSES') || 0; + my $pm = Parallel::ForkManager->new(FixMyStreet->test_mode ? 0 : $procs); while ( my $body = $bodies->next ) { + $pm->start and next; + $self->current_body( $body ); my %open311_conf = ( @@ -57,7 +62,11 @@ sub fetch { $self->blank_updates_permitted( $body->blank_updates_permitted ); $self->system_user( $body->comment_user ); $self->process_body(); + + $pm->finish; } + + $pm->wait_all_children; } sub parse_dates { |