diff options
| -rw-r--r-- | conf/general.yml-example | 3 | ||||
| -rw-r--r-- | cpanfile | 1 | ||||
| -rw-r--r-- | cpanfile.snapshot | 18 | ||||
| -rw-r--r-- | perllib/FixMyStreet.pm | 1 | ||||
| -rw-r--r-- | perllib/Open311/GetServiceRequestUpdates.pm | 9 |
5 files changed, 32 insertions, 0 deletions
diff --git a/conf/general.yml-example b/conf/general.yml-example index 8e8d0f76a..d7078a217 100644 --- a/conf/general.yml-example +++ b/conf/general.yml-example @@ -261,3 +261,6 @@ LOGIN_REQUIRED: 0 # If you want to stop new users from registering, set this to 1. # NB: This also disables all Facebook/Twitter logins. SIGNUPS_DISABLED: 0 + +# Setting this variable to more than 1 will let fetch-comments run in parallel +FETCH_COMMENTS_PROCESSES: 0 @@ -110,6 +110,7 @@ requires 'Net::OAuth'; requires 'Net::Twitter::Lite::WithAPIv1_1', '0.12008'; requires 'Number::Phone', '3.5000'; requires 'OIDC::Lite'; +requires 'Parallel::ForkManager'; requires 'Path::Class'; requires 'POSIX'; requires 'Readonly'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 42c125da6..8c55fec0c 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -6028,6 +6028,24 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.008001 + Parallel-ForkManager-2.02 + pathname: Y/YA/YANICK/Parallel-ForkManager-2.02.tar.gz + provides: + Parallel::ForkManager 2.02 + Parallel::ForkManager::Child 2.02 + requirements: + Carp 0 + ExtUtils::MakeMaker 0 + File::Path 0 + File::Spec 0 + File::Temp 0 + Moo 0 + Moo::Role 0 + POSIX 0 + Storable 0 + perl 5.006 + strict 0 + warnings 0 Params-Classify-0.015 pathname: Z/ZE/ZEFRAM/Params-Classify-0.015.tar.gz provides: diff --git a/perllib/FixMyStreet.pm b/perllib/FixMyStreet.pm index f6a69928b..94ba7685f 100644 --- a/perllib/FixMyStreet.pm +++ b/perllib/FixMyStreet.pm @@ -154,6 +154,7 @@ sub dbic_connect_info { my $dbi_args = { AutoCommit => 1, + AutoInactiveDestroy => 1, }; my $local_time_zone = local_time_zone(); my $dbic_args = { diff --git a/perllib/Open311/GetServiceRequestUpdates.pm b/perllib/Open311/GetServiceRequestUpdates.pm index a407d8e37..e27a08068 100644 --- a/perllib/Open311/GetServiceRequestUpdates.pm +++ b/perllib/Open311/GetServiceRequestUpdates.pm @@ -2,6 +2,7 @@ package Open311::GetServiceRequestUpdates; use Moo; use Open311; +use Parallel::ForkManager; use FixMyStreet::DB; use FixMyStreet::App::Model::PhotoSet; use DateTime::Format::W3CDTF; @@ -37,7 +38,11 @@ sub fetch { $bodies = $bodies->search( { name => $self->body } ); } + my $procs = FixMyStreet->config('FETCH_COMMENTS_PROCESSES') || 0; + my $pm = Parallel::ForkManager->new(FixMyStreet->test_mode ? 0 : $procs); while ( my $body = $bodies->next ) { + $pm->start and next; + $self->current_body( $body ); my %open311_conf = ( @@ -57,7 +62,11 @@ sub fetch { $self->blank_updates_permitted( $body->blank_updates_permitted ); $self->system_user( $body->comment_user ); $self->process_body(); + + $pm->finish; } + + $pm->wait_all_children; } sub parse_dates { |