first commit

This commit is contained in:
2025-02-24 22:33:42 +01:00
commit 737c037e85
18358 changed files with 5392983 additions and 0 deletions

View File

@@ -0,0 +1,286 @@
<?php
/**
* archive path file list object
*
* Standard: PSR-2
*
* @link http://www.php-fig.org/psr/psr-2
*
* @package name
* @copyright (c) 2019, Snapcreek LLC
* @license https://opensource.org/licenses/GPL-3.0 GNU Public License
*/
defined('ABSPATH') || defined('DUPXABSPATH') || exit;
use Duplicator\Libs\Snap\SnapIO;
use Duplicator\Libs\Snap\SnapJson;
class DUP_PRO_Archive_File_List
{
/** @var string */
protected $path = '';
/** @var ?resource */
protected $handle = null;
/** @var ?array<array{p:string,s:int,n:int}> */
protected $cache = null;
/**
* CLass constructor
*
* @param string $path path to file
*/
public function __construct($path)
{
if (empty($path)) {
throw new Exception('path can\'t be empty');
}
$this->path = SnapIO::safePath($path);
}
/**
* Class destructor
*
* @return void
*/
public function __destruct()
{
$this->close();
}
/**
* Get path
*
* @return string
*/
public function getPath()
{
return $this->path;
}
/**
* Open file
*
* @param bool $truncate if true truncate file
*
* @return bool
*/
public function open($truncate = false)
{
if (is_null($this->handle)) {
if (($this->handle = fopen($this->path, 'a+')) === false) {
DUP_PRO_Log::trace('Can\'t open ' . $this->path);
$this->handle = null;
return false;
}
}
if ($truncate) {
$this->emptyFile();
}
return true;
}
/**
* Empty file
*
* @return bool
*/
public function emptyFile()
{
if (!$this->open(false)) {
return false;
}
if (($res = ftruncate($this->handle, 0)) === false) {
DUP_PRO_Log::trace('Can\'t truncate file ' . $this->path);
return false;
}
return true;
}
/**
* Close file
*
* @return bool
*/
public function close()
{
if (!is_null($this->handle)) {
if (($res = @fclose($this->handle)) === false) {
DUP_PRO_Log::trace('Can\'t close ' . $this->path);
return false;
}
$this->handle = null;
}
return true;
}
/**
* Add entry
*
* @param string $path path
* @param int $size size
* @param int $nodes nodes
*
* @return bool true if success else false
*/
public function addEntry($path, $size, $nodes)
{
if (is_null($this->handle)) { // check to generate less overhead
if (!$this->open()) {
return false;
}
}
$entry = array(
'p' => $path,
's' => $size,
'n' => $nodes,
);
return (fwrite($this->handle, SnapJson::jsonEncode($entry) . "\n") !== false);
}
/**
* Get entry
*
* @param bool $pathOnly if true return only payth
*
* @return false|array{p:string,s:int,n:int}|string return false if is end of filer.
*/
public function getEntry($pathOnly = false)
{
if (is_null($this->handle)) { // check to generate less overhead
if (!$this->open()) {
return false;
}
}
if (($json = fgets($this->handle, 4196)) === false) {
// end of file return false
return false;
}
$result = json_decode($json, true);
if ($pathOnly) {
return $result['p'];
} else {
return $result;
}
}
/**
* Clean cache
*
* @return bool
*/
protected function cleanCache()
{
$this->cache = null;
return true;
}
/**
* Load cache
*
* @param bool $refreshCache if true refresh cache
*
* @return bool
*/
protected function loadCache($refreshCache = false)
{
if ($refreshCache || is_null($this->cache)) {
if (!$this->open()) {
return false;
}
$this->cache = array();
if (@fseek($this->handle, 0) === -1) {
DUP_PRO_Log::trace('Can\'t seek at 0 pos for file ' . $this->path);
$this->cleanCache();
return false;
}
while (($entry = $this->getEntry()) !== false) {
$this->cache[$entry['p']] = $entry;
}
if (!feof($this->handle)) {
DUP_PRO_Log::trace('Error: unexpected fgets() fail', false);
}
}
return true;
}
/**
* Get entry from path
*
* @param string $path path
* @param bool $refreshCache if true refresh cache
*
* @return bool|array{p:string,s:int,n:int}
*/
public function getEntryFromPath($path, $refreshCache = false)
{
if (!$this->loadCache($refreshCache)) {
return false;
}
if (array_key_exists($path, $this->cache)) {
return $this->cache[$path];
} else {
return false;
}
}
/**
* Get entries from path
*
* @param string $path path
* @param bool $refreshCache if true refresh cache
*
* @return bool|array<array{p:string,s:int,n:int}>
*/
public function getEntriesFormPath($path, $refreshCache = false)
{
if (!$this->loadCache($refreshCache)) {
return false;
}
if (array_key_exists($path, $this->cache)) {
$result = array();
foreach ($this->cache as $current => $entry) {
if (preg_match('/^' . preg_quote($path, '/') . '\/[^\/]+$/', $current)) {
$result[] = $entry;
}
}
return $result;
} else {
return false;
}
}
/**
* Get array paths
*
* @param string $pathPrefix path prefix
*
* @return bool|string[]
*/
public function getArrayPaths($pathPrefix = '')
{
if (!$this->open()) {
return false;
}
$result = array();
if (@fseek($this->handle, 0) === -1) {
DUP_PRO_Log::trace('Can\'t seek at 0 pos for file ' . $this->path);
return false;
}
$safePrefix = SnapIO::safePathUntrailingslashit($pathPrefix);
while (($path = $this->getEntry(true)) !== false) {
$result[] = $safePrefix . '/' . $path;
}
if (!feof($this->handle)) {
DUP_PRO_Log::trace('Error: unexpected fgets() fail', false);
}
return $result;
}
}

View File

@@ -0,0 +1,187 @@
<?php
defined('ABSPATH') || defined('DUPXABSPATH') || exit;
/**
* Defines the scope from which a filter item was created/retrieved from
*
* @package DupicatorPro\classes
*/
class DUP_PRO_Archive_Filter_Scope_Base
{
/** @var string[] All internal storage items that we decide to filter */
public $Core = array();
//TODO: Enable with Settings UI
/** @var string[] Global filter items added from settings */
public $Global = array();
/** @var string[] Items when creating a package or template */
public $Instance = array();
/** @var string[] Items that are not readable */
public $Unreadable = array();
/** @var int Number of unreadable items */
private $unreadableCount = 0;
/**
* Filter props on json encode
*
* @return string[]
*/
public function __sleep()
{
$props = array_keys(get_object_vars($this));
return array_diff($props, array('unreadableCount'));
}
/**
* @param string $item A path to an unreadable item
*
* @return void
*/
public function addUnreadableItem($item)
{
$this->unreadableCount++;
if ($this->unreadableCount <= DUPLICATOR_PRO_SCAN_MAX_UNREADABLE_COUNT) {
$this->Unreadable[] = $item;
}
}
/**
* @return int returns number of unreadable items
*/
public function getUnreadableCount()
{
return $this->unreadableCount;
}
}
/**
* Defines the scope from which a filter item was created/retrieved from
*
* @package DupicatorPro\classes
*/
class DUP_PRO_Archive_Filter_Scope_Directory extends DUP_PRO_Archive_Filter_Scope_Base
{
/**
* @var string[] Directories containing other WordPress installs
*/
public $AddonSites = array();
/**
* @var array<array<string,mixed>> Items that are too large
*/
public $Size = array();
}
/**
* Defines the scope from which a filter item was created/retrieved from
*
* @package DupicatorPro\classes
*/
class DUP_PRO_Archive_Filter_Scope_File extends DUP_PRO_Archive_Filter_Scope_Base
{
/**
* @var array<array<string,mixed>> Items that are too large
*/
public $Size = array();
}
/**
* Defines the filtered items that are pulled from there various scopes
*
* @package DupicatorPro\classes
*/
class DUP_PRO_Archive_Filter_Info
{
/** @var ?DUP_PRO_Archive_Filter_Scope_Directory Contains all folder filter info */
public $Dirs = null;
/** @var ?DUP_PRO_Archive_Filter_Scope_File Contains all folder filter info */
public $Files = null;
/** @var ?DUP_PRO_Archive_Filter_Scope_Base Contains all folder filter info */
public $Exts = null;
/** @var null|array<string,mixed>|DUP_PRO_Tree_files tree size structure for client jstree */
public $TreeSize = null;
/**
* Class constructor
*/
public function __construct()
{
$this->reset(true);
}
/**
* Clone
*
* @return void
*/
public function __clone()
{
if (is_object($this->Dirs)) {
$this->Dirs = clone $this->Dirs;
}
if (is_object($this->Files)) {
$this->Files = clone $this->Files;
}
if (is_object($this->Exts)) {
$this->Exts = clone $this->Exts;
}
if (is_object($this->TreeSize)) {
$this->TreeSize = clone $this->TreeSize;
}
}
/**
* reset and clean all object
*
* @param bool $initTreeObjs if true then init tree size object
*
* @return void
*/
public function reset($initTreeObjs = false)
{
$exclude = array(
"Unreadable",
"Instance",
);
if (is_null($this->Dirs)) {
$this->Dirs = new DUP_PRO_Archive_Filter_Scope_Directory();
} else {
$this->resetMember($this->Dirs, $exclude);
}
if (is_null($this->Files)) {
$this->Files = new DUP_PRO_Archive_Filter_Scope_File();
} else {
$this->resetMember($this->Files, $exclude);
}
$this->Exts = new DUP_PRO_Archive_Filter_Scope_Base();
if ($initTreeObjs) {
$this->TreeSize = new DUP_PRO_Tree_files(ABSPATH, false);
} else {
$this->TreeSize = null;
}
}
/**
* Resets all properties of $member to their default values except the ones in $exclude
*
* @param object $member Object to reset
* @param string[] $exclude Properties to exclude from resetting
*
* @return void
*/
private function resetMember($member, $exclude = array())
{
$refClass = new ReflectionClass($member);
$defaults = $refClass->getDefaultProperties();
foreach ($member as $key => $value) {
if (!in_array($key, $exclude)) {
if (isset($defaults[$key])) {
$member->$key = $defaults[$key];
} else {
$member->$key = null;
}
}
}
}
}

View File

@@ -0,0 +1,266 @@
<?php
defined('ABSPATH') || defined('DUPXABSPATH') || exit;
use Duplicator\Libs\Snap\SnapIO;
use Duplicator\Libs\Shell\Shell;
use Duplicator\Models\Storages\AbstractStorageEntity;
use Duplicator\Models\Storages\Local\LocalStorage;
use Duplicator\Models\SystemGlobalEntity;
use Duplicator\Package\Create\BuildProgress;
/**
* Creates a zip file using Shell_Exec and the system zip command
* Not available on all system
**/
class DUP_PRO_ShellZip
{
/**
* Creates the zip file and adds the SQL file to the archive
*
* @param DUP_PRO_Package $package The package object
* @param BuildProgress $build_progress The build progress object
*
* @return boolean
*/
public static function create(DUP_PRO_Package $package, BuildProgress $build_progress)
{
$archive = $package->Archive;
try {
if ($package->Status == DUP_PRO_PackageStatus::ARCSTART) {
$error_text = __('Zip process getting killed due to limited server resources.', 'duplicator-pro');
$fix_text = __('Click to switch Archive Engine DupArchive.', 'duplicator-pro');
$system_global = SystemGlobalEntity::getInstance();
$system_global->addQuickFix(
$error_text,
$fix_text,
array(
'global' => array('archive_build_mode' => 3),
)
);
DUP_PRO_Log::traceError("$error_text **RECOMMENDATION: $fix_text");
if ($build_progress->retries > 1) {
$build_progress->failed = true;
return true;
} else {
$build_progress->retries++;
$package->update();
}
}
$package->set_status(DUP_PRO_PackageStatus::ARCSTART);
$package->safe_tmp_cleanup(true);
$compressDir = rtrim(SnapIO::safePath($archive->PackDir), '/');
$zipPath = SnapIO::safePath("{$package->StorePath}/{$archive->File}");
$sql_filepath = SnapIO::safePath("{$package->StorePath}/{$package->Database->File}");
$filterDirs = empty($archive->FilterDirs) ? 'not set' : rtrim(str_replace(';', "\n\t", $archive->FilterDirs));
$filterFiles = empty($archive->FilterFiles) ? 'not set' : rtrim(str_replace(';', "\n\t", $archive->FilterFiles));
$filterExts = empty($archive->FilterExts) ? 'not set' : $archive->FilterExts;
$filterOn = ($archive->FilterOn) ? 'ON' : 'OFF';
$scanFilepath = DUPLICATOR_PRO_SSDIR_PATH_TMP . "/{$package->NameHash}_scan.json";
// LOAD SCAN REPORT
try {
$scanReport = $package->getScanReportFromJson($scanFilepath);
} catch (DUP_PRO_NoScanFileException $ex) {
DUP_PRO_Log::trace("**** scan file $scanFilepath doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
} catch (DUP_PRO_NoFileListException $ex) {
DUP_PRO_Log::trace("**** list of files doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
} catch (DUP_PRO_NoDirListException $ex) {
DUP_PRO_Log::trace("**** list of directories doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
}
DUP_PRO_Log::info("\n********************************************************************************");
DUP_PRO_Log::info("ARCHIVE Type=ZIP Mode=Shell");
DUP_PRO_Log::info("********************************************************************************");
DUP_PRO_Log::info("ARCHIVE DIR: " . $compressDir);
DUP_PRO_Log::info("ARCHIVE FILE: " . basename($zipPath));
DUP_PRO_Log::info("FILTERS: *{$filterOn}*");
DUP_PRO_Log::info("DIRS: {$filterDirs}");
DUP_PRO_Log::info("EXTS: {$filterExts}");
DUP_PRO_Log::info("FILES: {$filterFiles}");
DUP_PRO_Log::info("----------------------------------------");
DUP_PRO_Log::info("COMPRESSING");
DUP_PRO_Log::info("SIZE:\t" . $scanReport->ARC->Size);
DUP_PRO_Log::info("STATS:\tDirs " . $scanReport->ARC->DirCount . " | Files " . $scanReport->ARC->FileCount . " | Total " . $scanReport->ARC->FullCount);
$build_progress->archive_started = true;
$build_progress->archive_start_time = DUP_PRO_U::getMicrotime();
$contains_root = false;
$exclude_string = '';
$filterDirs = $archive->FilterDirsAll;
$filterExts = $archive->FilterExtsAll;
$filterFiles = $archive->FilterFilesAll;
// DIRS LIST
foreach ($filterDirs as $filterDir) {
if (trim($filterDir) != '') {
$relative_filter_dir = DUP_PRO_U::getRelativePath($compressDir, $filterDir);
DUP_PRO_Log::trace("Adding relative filter dir $relative_filter_dir for $filterDir relative to $compressDir");
if (trim($relative_filter_dir) == '') {
$contains_root = true;
break;
} else {
$exclude_string .= DUP_PRO_Zip_U::customShellArgEscapeSequence($relative_filter_dir) . "**\* ";
$exclude_string .= DUP_PRO_Zip_U::customShellArgEscapeSequence($relative_filter_dir) . " ";
}
}
}
//EXT LIST
foreach ($filterExts as $filterExt) {
$exclude_string .= "\*.$filterExt ";
}
//FILE LIST
foreach ($filterFiles as $filterFile) {
if (trim($filterFile) != '') {
$relative_filter_file = DUP_PRO_U::getRelativePath($compressDir, trim($filterFile));
DUP_PRO_Log::trace("Full file=$filterFile relative=$relative_filter_file compressDir=$compressDir");
$exclude_string .= "\"$relative_filter_file\" ";
}
}
//DB ONLY
if ($package->isDBOnly()) {
$contains_root = true;
}
if ($contains_root == false) {
// Only attempt to zip things up if root isn't in there since stderr indicates when it cant do anything
$storages = AbstractStorageEntity::getAll();
foreach ($storages as $storage) {
if ($storage->getSType() !== LocalStorage::getSType()) {
continue;
}
/** @var LocalStorage $storage */
if ($storage->isFilterProtection()) {
continue;
}
$storagePath = SnapIO::safePath($storage->getLocationString());
$storagePath = DUP_PRO_U::getRelativePath($compressDir, $storagePath);
$exclude_string .= "$storagePath**\* ";
}
$relative_backup_dir = DUP_PRO_U::getRelativePath($compressDir, DUPLICATOR_PRO_SSDIR_PATH);
$exclude_string .= "$relative_backup_dir**\* ";
$params = Shell::getCompressionParam($build_progress->current_build_compression);
if (strlen($package->Archive->getArchivePassword()) > 0) {
$params .= ' --password ' . escapeshellarg($package->Archive->getArchivePassword());
}
$params .= ' -rq';
$command = 'cd ' . escapeshellarg($compressDir);
$command .= ' && ' . escapeshellcmd(DUP_PRO_Zip_U::getShellExecZipPath()) . ' ' . $params . ' ';
$command .= escapeshellarg($zipPath) . ' ./';
$command .= " -x $exclude_string 2>&1";
DUP_PRO_Log::infoTrace("SHELL COMMAND: $command");
$shellOutput = Shell::runCommand($command, Shell::AVAILABLE_COMMANDS);
DUP_PRO_Log::trace("After shellzip command");
if ($shellOutput !== false && !$shellOutput->isEmpty()) {
$stderr = $shellOutput->getOutputAsString();
$error_text = "Error executing shell exec zip: $stderr.";
$system_global = SystemGlobalEntity::getInstance();
if (DUP_PRO_STR::contains($stderr, 'quota')) {
$fix_text = __("Account out of space so purge large files or talk to your host about increasing quota.", 'duplicator-pro');
$system_global->addTextFix($error_text, $fix_text);
} elseif (DUP_PRO_STR::contains($stderr, 'such file or')) {
$fix_text = sprintf(
"%s <a href='" . DUPLICATOR_PRO_DUPLICATOR_DOCS_URL . "how-to-resolve-zip-format-related-build-issues' target='_blank'>%s</a>",
__('See FAQ:', 'duplicator-pro'),
__('How to resolve "zip warning: No such file or directory"?', 'duplicator-pro')
);
$system_global->addTextFix($error_text, $fix_text);
} else {
$fix_text = __("Click on button to switch to the DupArchive engine.", 'duplicator-pro');
$system_global->addQuickFix(
$error_text,
$fix_text,
array(
'global' => array('archive_build_mode' => 3),
)
);
}
DUP_PRO_Log::error("$error_text **RECOMMENDATION: $fix_text", '', false);
$build_progress->failed = true;
return true;
} else {
DUP_PRO_Log::trace("Stderr is null");
}
$file_count_string = '';
if (!file_exists($zipPath)) {
$file_count_string = sprintf(__('Zip file %s does not exist.', 'duplicator-pro'), $zipPath);
} elseif (DUP_PRO_U::getExeFilepath('zipinfo') != null) {
DUP_PRO_Log::trace("zipinfo exists");
$file_count_string = "zipinfo -t '$zipPath'";
} elseif (DUP_PRO_U::getExeFilepath('unzip') != null) {
DUP_PRO_Log::trace("zipinfo doesn't exist so reverting to unzip");
$file_count_string = "unzip -l '$zipPath' | wc -l";
}
if ($file_count_string != '') {
$shellOutput = Shell::runCommand($file_count_string . ' | awk \'{print $1 }\'', Shell::AVAILABLE_COMMANDS);
$file_count = ($shellOutput !== false)
? trim($shellOutput->getOutputAsString())
: null;
if (is_numeric($file_count)) {
// Accounting for the sql and installer back files
$archive->file_count = (int) $file_count + 2;
} else {
$error_text = sprintf(
__("Error retrieving file count in shell zip %s.", 'duplicator-pro'),
$file_count_string
);
DUP_PRO_Log::trace("Executed file count string of $file_count_string");
DUP_PRO_Log::trace($error_text);
$fix_text = __("Click on button to switch to the DupArchive engine.", 'duplicator-pro');
$system_global = SystemGlobalEntity::getInstance();
$system_global->addQuickFix(
$error_text,
$fix_text,
array(
'global' => array('archive_build_mode' => 3),
)
);
DUP_PRO_Log::error("$error_text **RECOMMENDATION:$fix_text", '', false);
DUP_PRO_Log::trace("$error_text **RECOMMENDATION:$fix_text");
$build_progress->failed = true;
$archive->file_count = -2;
return true;
}
} else {
DUP_PRO_Log::trace("zipinfo doesn't exist");
// The -1 and -2 should be constants since they signify different things
$archive->file_count = -1;
}
} else {
$archive->file_count = 2;
// Installer bak and database.sql
}
DUP_PRO_Log::trace("archive file count from shellzip is $archive->file_count");
$build_progress->archive_built = true;
$build_progress->retries = 0;
$package->update();
$timerAllEnd = DUP_PRO_U::getMicrotime();
$timerAllSum = DUP_PRO_U::elapsedTime($timerAllEnd, $build_progress->archive_start_time);
$zipFileSize = @filesize($zipPath);
DUP_PRO_Log::info("COMPRESSED SIZE: " . DUP_PRO_U::byteSize($zipFileSize));
DUP_PRO_Log::info("ARCHIVE RUNTIME: {$timerAllSum}");
DUP_PRO_Log::info("MEMORY STACK: " . DUP_PRO_Server::getPHPMemory());
} catch (Exception $e) {
DUP_PRO_Log::error("Runtime error in shell exec zip compression.", "Exception: {$e}");
}
return true;
}
}

View File

@@ -0,0 +1,665 @@
<?php
/**
* Class to create a zip file using PHP ZipArchive
*
* Standard: PSR-2 (almost)
*
* @link http://www.php-fig.org/psr/psr-2
*
* @package DUP_PRO
* @subpackage classes/package
* @copyright (c) 2017, Snapcreek LLC
* @license https://opensource.org/licenses/GPL-3.0 GNU Public License
* @since 1.0.0
*
* @notes: Trace process time
* $timer01 = DUP_PRO_U::getMicrotime();
* DUP_PRO_Log::trace("SCAN TIME-B = " . DUP_PRO_U::elapsedTime(DUP_PRO_U::getMicrotime(), $timer01));
*/
defined('ABSPATH') || defined('DUPXABSPATH') || exit;
use Duplicator\Libs\Snap\SnapIO;
use Duplicator\Models\SystemGlobalEntity;
use Duplicator\Package\Create\BuildProgress;
use Duplicator\Utils\ZipArchiveExtended;
class DUP_PRO_ZipArchive
{
/** @var DUP_PRO_Global_Entity */
private $global = null;
/** @var bool */
private $optMaxBuildTimeOn = true;
/** @var int */
private $maxBuildTimeFileSize = 100000;
/** @var int */
private $throttleDelayInUs = 0;
/** @var DUP_PRO_Package */
private $package = null;
/** @var ZipArchiveExtended */
private $zipArchive = null;
/**
* Class constructor
*
* @param DUP_PRO_Package $package The package to create the zip file for
*/
public function __construct(DUP_PRO_Package $package)
{
$this->global = DUP_PRO_Global_Entity::getInstance();
$this->optMaxBuildTimeOn = ($this->global->max_package_runtime_in_min > 0);
$this->throttleDelayInUs = $this->global->getMicrosecLoadReduction();
$this->package = $package;
$this->zipArchive = new ZipArchiveExtended($this->package->StorePath . '/' . $this->package->Archive->File);
$password = $this->package->Archive->getArchivePassword();
if (strlen($password) > 0) {
$this->zipArchive->setEncrypt(true, $password);
}
}
/**
* Creates the zip file and adds the SQL file to the archive
*
* @param BuildProgress $build_progress A copy of the current build progress
*
* @return bool Returns true if the process was successful
*/
public function create(BuildProgress $build_progress)
{
try {
if (!ZipArchiveExtended::isPhpZipAvailable()) {
DUP_PRO_Log::trace("Zip archive doesn't exist?");
return false;
}
$this->package->safe_tmp_cleanup(true);
if ($this->package->ziparchive_mode == DUP_PRO_ZipArchive_Mode::SingleThread) {
return $this->createSingleThreaded($build_progress);
} else {
return $this->createMultiThreaded($build_progress);
}
} catch (Exception $ex) {
DUP_PRO_Log::error("Runtime error in class-package-archive-zip.php.", "Exception: {$ex}");
return false;
}
}
/**
* Creates the zip file using a single thread approach
*
* @param BuildProgress $build_progress A copy of the current build progress
*
* @return bool Returns true if the process was successful
*/
private function createSingleThreaded(BuildProgress $build_progress)
{
$countFiles = 0;
$compressDir = rtrim(SnapIO::safePath($this->package->Archive->PackDir), '/');
$sqlPath = $this->package->StorePath . '/' . $this->package->Database->File;
$zipPath = $this->package->StorePath . '/' . $this->package->Archive->File;
$filterDirs = empty($this->package->Archive->FilterDirs) ? 'not set' : rtrim(str_replace(';', "\n\t", $this->package->Archive->FilterDirs));
$filterFiles = empty($this->package->Archive->FilterFiles) ? 'not set' : rtrim(str_replace(';', "\n\t", $this->package->Archive->FilterFiles));
$filterExts = empty($this->package->Archive->FilterExts) ? 'not set' : $this->package->Archive->FilterExts;
$filterOn = ($this->package->Archive->FilterOn) ? 'ON' : 'OFF';
$validation = ($this->global->ziparchive_validation) ? 'ON' : 'OFF';
$compression = $build_progress->current_build_compression ? 'ON' : 'OFF';
$this->zipArchive->setCompressed($build_progress->current_build_compression);
//PREVENT RETRIES PAST 3: Default is 10 (DUP_PRO_Constants::MAX_BUILD_RETRIES)
//since this is ST Mode no reason to keep trying like MT
if ($build_progress->retries >= 3) {
$err = __('Package build appears stuck so marking package as failed. Is the PHP or Web Server timeouts too low?', 'duplicator-pro');
DUP_PRO_Log::error(__('Build Failure', 'duplicator-pro'), $err, false);
DUP_PRO_Log::trace($err);
return $build_progress->failed = true;
} else {
if ($build_progress->retries > 0) {
DUP_PRO_Log::infoTrace("**NOTICE: Retry count at: {$build_progress->retries}");
}
$build_progress->retries++;
$this->package->update();
}
//LOAD SCAN REPORT
try {
$scanReport = $this->package->getScanReportFromJson(DUPLICATOR_PRO_SSDIR_PATH_TMP . "/{$this->package->NameHash}_scan.json");
} catch (DUP_PRO_NoScanFileException $ex) {
DUP_PRO_Log::trace("**** scan file doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
} catch (DUP_PRO_NoFileListException $ex) {
DUP_PRO_Log::trace("**** list of files doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
} catch (DUP_PRO_NoDirListException $ex) {
DUP_PRO_Log::trace("**** list of directories doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
}
//============================================
//ST: START ZIP
//============================================
if ($build_progress->archive_started === false) {
DUP_PRO_Log::info("\n********************************************************************************");
DUP_PRO_Log::info("ARCHIVE ZipArchive Single-Threaded");
DUP_PRO_Log::info("********************************************************************************");
DUP_PRO_Log::info("ARCHIVE DIR: " . $compressDir);
DUP_PRO_Log::info("ARCHIVE FILE: " . basename($zipPath));
DUP_PRO_Log::info("COMPRESSION: *{$compression}*");
DUP_PRO_Log::info("VALIDATION: *{$validation}*");
DUP_PRO_Log::info("FILTERS: *{$filterOn}*");
DUP_PRO_Log::info("DIRS:\t{$filterDirs}");
DUP_PRO_Log::info("EXTS: {$filterExts}");
DUP_PRO_Log::info("FILES: {$filterFiles}");
DUP_PRO_Log::info("----------------------------------------");
DUP_PRO_Log::info("COMPRESSING");
DUP_PRO_Log::info("SIZE:\t" . $scanReport->ARC->Size);
DUP_PRO_Log::info("STATS:\tDirs " . $scanReport->ARC->DirCount . " | Files " . $scanReport->ARC->FileCount . " | Total " . $scanReport->ARC->FullCount);
if (($scanReport->ARC->DirCount == '') || ($scanReport->ARC->FileCount == '') || ($scanReport->ARC->FullCount == '')) {
DUP_PRO_Log::error('Invalid Scan Report Detected', 'Invalid Scan Report Detected', false);
return $build_progress->failed = true;
}
$build_progress->archive_started = true;
$build_progress->archive_start_time = DUP_PRO_U::getMicrotime();
}
//============================================
//ST: ADD DATABASE FILE
//============================================
if ($build_progress->archive_has_database === false) {
if (!$this->zipArchive->open()) {
DUP_PRO_Log::error("Couldn't open $zipPath", '', false);
return $build_progress->failed = true;
}
if ($this->zipArchive->addFile($sqlPath, $this->package->get_sql_ark_file_path())) {
DUP_PRO_Log::info("SQL ADDED: " . basename($sqlPath));
} else {
DUP_PRO_Log::error("Unable to add database.sql to archive.", "SQL File Path [" . $sqlPath . "]", false);
return $build_progress->failed = true;
}
if ($this->zipArchive->close()) {
$build_progress->archive_has_database = true;
$this->package->update();
} else {
$err = 'ZipArchive close failure during database.sql phase.';
$this->setDupArchiveSwitchFix($err);
return $build_progress->failed = true;
}
}
//============================================
//ST: ZIP DIRECTORIES
//Keep this loop tight: ZipArchive can handle over 10k+ dir entries in under 0.01 seconds.
//Its really fast without files so no need to do status pushes or other checks in loop
//============================================
if ($build_progress->next_archive_dir_index < count($scanReport->ARC->Dirs)) {
if (!$this->zipArchive->open()) {
DUP_PRO_Log::error("Couldn't open $zipPath", '', false);
return $build_progress->failed = true;
}
foreach ($scanReport->ARC->Dirs as $dir) {
$emptyDir = $this->package->Archive->getLocalDirPath($dir);
DUP_PRO_Log::trace("ADD DIR TO ZIP: '{$emptyDir}'");
if (!$this->zipArchive->addEmptyDir($emptyDir)) {
if (empty($compressDir) || strpos($dir, rtrim($compressDir, '/')) != 0) {
DUP_PRO_Log::infoTrace("WARNING: Unable to zip directory: '{$dir}'");
}
}
$build_progress->next_archive_dir_index++;
}
if ($this->zipArchive->close()) {
$this->package->update();
} else {
$err = 'ZipArchive close failure during directory add phase.';
$this->setDupArchiveSwitchFix($err);
return $build_progress->failed = true;
}
}
//============================================
//ST: ZIP FILES
//============================================
if ($build_progress->archive_built === false) {
if ($this->zipArchive->open() === false) {
DUP_PRO_Log::error("Can not open zip file at: [{$zipPath}]", '', false);
return $build_progress->failed = true;
}
// Since we have to estimate progress in Single Thread mode
// set the status when we start archiving just like Shell Exec
$this->package->set_status(DUP_PRO_PackageStatus::ARCSTART);
$total_file_size = 0;
$total_file_count_trip = ($scanReport->ARC->UFileCount + 1000);
foreach ($scanReport->ARC->Files as $file) {
//NON-ASCII check
if (preg_match('/[^\x20-\x7f]/', $file)) {
if (!$this->isUTF8FileSafe($file)) {
continue;
}
}
if ($this->global->ziparchive_validation) {
if (!is_readable($file)) {
DUP_PRO_Log::infoTrace("NOTICE: File [{$file}] is unreadable!");
continue;
}
}
$local_name = $this->package->Archive->getLocalFilePath($file);
if (!$this->zipArchive->addFile($file, $local_name)) {
// Assumption is that we continue?? for some things this would be fatal others it would be ok - leave up to user
DUP_PRO_Log::info("WARNING: Unable to zip file: {$file}");
continue;
}
$total_file_size += filesize($file);
//ST: SERVER THROTTLE
if ($this->throttleDelayInUs !== 0) {
usleep($this->throttleDelayInUs);
}
//Prevent Overflow
if ($countFiles++ > $total_file_count_trip) {
DUP_PRO_Log::error("ZipArchive-ST: file loop overflow detected at {$countFiles}", '', false);
return $build_progress->failed = true;
}
}
//START ARCHIVE CLOSE
$total_file_size_easy = DUP_PRO_U::byteSize($total_file_size);
DUP_PRO_Log::trace("Doing final zip close after adding $total_file_size_easy ({$total_file_size})");
if ($this->zipArchive->close()) {
DUP_PRO_Log::trace("Final zip closed.");
$build_progress->next_archive_file_index = $countFiles;
$build_progress->archive_built = true;
$this->package->update();
} else {
if ($this->global->ziparchive_validation === false) {
$this->global->ziparchive_validation = true;
$this->global->save();
DUP_PRO_Log::infoTrace("**NOTICE: ZipArchive: validation mode enabled");
} else {
$err = 'ZipArchive close failure during file phase with file validation enabled';
$this->setDupArchiveSwitchFix($err);
return $build_progress->failed = true;
}
}
}
//============================================
//ST: LOG FINAL RESULTS
//============================================
if ($build_progress->archive_built) {
$timerAllEnd = DUP_PRO_U::getMicrotime();
$timerAllSum = DUP_PRO_U::elapsedTime($timerAllEnd, $build_progress->archive_start_time);
$zipFileSize = @filesize($zipPath);
DUP_PRO_Log::info("MEMORY STACK: " . DUP_PRO_Server::getPHPMemory());
DUP_PRO_Log::info("FINAL SIZE: " . DUP_PRO_U::byteSize($zipFileSize));
DUP_PRO_Log::info("ARCHIVE RUNTIME: {$timerAllSum}");
if ($this->zipArchive->open()) {
$this->package->Archive->file_count = $this->zipArchive->getNumFiles();
$this->package->update();
$this->zipArchive->close();
} else {
DUP_PRO_Log::error("ZipArchive open failure.", "Encountered when retrieving final archive file count.", false);
return $build_progress->failed = true;
}
}
return true;
}
/**
* Creates the zip file using a multi-thread approach
*
* @param BuildProgress $build_progress A copy of the current build progress
*
* @return bool Returns true if the process was successful
*/
private function createMultiThreaded(BuildProgress $build_progress)
{
$timed_out = false;
$countFiles = 0;
$compressDir = rtrim(SnapIO::safePath($this->package->Archive->PackDir), '/');
$sqlPath = $this->package->StorePath . '/' . $this->package->Database->File;
$zipPath = $this->package->StorePath . '/' . $this->package->Archive->File;
$filterDirs = empty($this->package->Archive->FilterDirs) ? 'not set' : rtrim(str_replace(';', "\n\t", $this->package->Archive->FilterDirs));
$filterFiles = empty($this->package->Archive->FilterFiles) ? 'not set' : rtrim(str_replace(';', "\n\t", $this->package->Archive->FilterFiles));
$filterExts = empty($this->package->Archive->FilterExts) ? 'not set' : $this->package->Archive->FilterExts;
$filterOn = ($this->package->Archive->FilterOn) ? 'ON' : 'OFF';
$compression = $build_progress->current_build_compression ? 'ON' : 'OFF';
$this->zipArchive->setCompressed($build_progress->current_build_compression);
$scanFilepath = DUPLICATOR_PRO_SSDIR_PATH_TMP . "/{$this->package->NameHash}_scan.json";
//LOAD SCAN REPORT
try {
$scanReport = $this->package->getScanReportFromJson($scanFilepath);
} catch (DUP_PRO_NoScanFileException $ex) {
DUP_PRO_Log::trace("**** scan file $scanFilepath doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
} catch (DUP_PRO_NoFileListException $ex) {
DUP_PRO_Log::trace("**** list of files doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
} catch (DUP_PRO_NoDirListException $ex) {
DUP_PRO_Log::trace("**** list of directories doesn't exist!!");
DUP_PRO_Log::error($ex->getMessage(), '', false);
$build_progress->failed = true;
return true;
}
//============================================
//MT: START ZIP & ADD SQL FILE
//============================================
if ($build_progress->archive_started === false) {
DUP_PRO_Log::info("\n********************************************************************************");
DUP_PRO_Log::info("ARCHIVE Mode:ZipArchive Multi-Threaded");
DUP_PRO_Log::info("********************************************************************************");
DUP_PRO_Log::info("ARCHIVE DIR: " . $compressDir);
DUP_PRO_Log::info("ARCHIVE FILE: " . basename($zipPath));
DUP_PRO_Log::info("COMPRESSION: *{$compression}*");
DUP_PRO_Log::info("FILTERS: *{$filterOn}*");
DUP_PRO_Log::info("DIRS: {$filterDirs}");
DUP_PRO_Log::info("EXTS: {$filterExts}");
DUP_PRO_Log::info("FILES: {$filterFiles}");
DUP_PRO_Log::info("----------------------------------------");
DUP_PRO_Log::info("COMPRESSING");
DUP_PRO_Log::info("SIZE:\t" . $scanReport->ARC->Size);
DUP_PRO_Log::info("STATS:\tDirs " . $scanReport->ARC->DirCount . " | Files " . $scanReport->ARC->FileCount . " | Total " . $scanReport->ARC->FullCount);
if (($scanReport->ARC->DirCount == '') || ($scanReport->ARC->FileCount == '') || ($scanReport->ARC->FullCount == '')) {
DUP_PRO_Log::error('Invalid Scan Report Detected', 'Invalid Scan Report Detected', false);
return $build_progress->failed = true;
}
if (!$this->zipArchive->open()) {
DUP_PRO_Log::error("Couldn't open $zipPath", '', false);
return $build_progress->failed = true;
}
if ($this->zipArchive->addFile($sqlPath, $this->package->get_sql_ark_file_path())) {
DUP_PRO_Log::info("SQL ADDED: " . basename($sqlPath));
} else {
DUP_PRO_Log::error("Unable to add database.sql to archive.", "SQL File Path [" . $sqlPath . "]", false);
return $build_progress->failed = true;
}
if ($this->zipArchive->close()) {
$build_progress->archive_has_database = true;
$this->package->update();
} else {
$err = 'ZipArchive close failure during database.sql phase.';
$this->setDupArchiveSwitchFix($err);
return $build_progress->failed = true;
}
}
//============================================
//MT: ZIP DIRECTORIES
//Keep this loop tight: ZipArchive can handle over 10k dir entries in under 0.01 seconds.
//Its really fast without files no need to do status pushes or other checks in loop
//============================================
if ($this->zipArchive->open()) {
foreach ($scanReport->ARC->Dirs as $dir) {
$emptyDir = $this->package->Archive->getLocalDirPath($dir);
DUP_PRO_Log::trace("ADD DIR TO ZIP: '{$emptyDir}'");
if (!$this->zipArchive->addEmptyDir($emptyDir)) {
if (empty($compressDir) || strpos($dir, rtrim($compressDir, '/')) != 0) {
DUP_PRO_Log::infoTrace("WARNING: Unable to zip directory: '{$dir}'");
}
}
$build_progress->next_archive_dir_index++;
}
$this->package->update();
if ($build_progress->timedOut($this->global->php_max_worker_time_in_sec)) {
$timed_out = true;
$diff = time() - $build_progress->thread_start_time;
DUP_PRO_Log::trace("Timed out after hitting thread time of $diff {$this->global->php_max_worker_time_in_sec} so quitting zipping early in the directory phase");
}
} else {
DUP_PRO_Log::error("Couldn't open $zipPath", '', false);
return $build_progress->failed = true;
}
if ($this->zipArchive->close() === false) {
$err = __('ZipArchive close failure during directory add phase.', 'duplicator-pro');
$this->setDupArchiveSwitchFix($err);
return $build_progress->failed = true;
}
//============================================
//MT: ZIP FILES
//============================================
if ($timed_out === false) {
// PREVENT RETRIES (10x)
if ($build_progress->retries > DUP_PRO_Constants::MAX_BUILD_RETRIES) {
$err = __('Zip build appears stuck.', 'duplicator-pro');
$this->setDupArchiveSwitchFix($err);
$error_msg = __('Package build appears stuck so marking package failed. Recommend setting Settings > Packages > Archive Engine to DupArchive', 'duplicator-pro');
DUP_PRO_Log::error(__('Build Failure', 'duplicator-pro'), $error_msg, false);
DUP_PRO_Log::trace($error_msg);
return $build_progress->failed = true;
} else {
$build_progress->retries++;
$this->package->update();
}
$zip_is_open = false;
$total_file_size = 0;
$incremental_file_size = 0;
$used_zip_file_descriptor_count = 0;
$total_file_count = empty($scanReport->ARC->UFileCount) ? 0 : $scanReport->ARC->UFileCount;
foreach ($scanReport->ARC->Files as $file) {
if ($zip_is_open || ($countFiles == $build_progress->next_archive_file_index)) {
if ($zip_is_open === false) {
DUP_PRO_Log::trace("resuming archive building at file # $countFiles");
if ($this->zipArchive->open() !== true) {
DUP_PRO_Log::error("Couldn't open $zipPath", '', false);
$build_progress->failed = true;
return true;
}
$zip_is_open = true;
}
//NON-ASCII check
if (preg_match('/[^\x20-\x7f]/', $file)) {
if (!$this->isUTF8FileSafe($file)) {
continue;
}
} elseif (!file_exists($file)) {
DUP_PRO_Log::trace("NOTICE: ASCII file [{$file}] does not exist!");
continue;
}
$local_name = $this->package->Archive->getLocalFilePath($file);
$file_size = filesize($file);
$zip_status = $this->zipArchive->addFile($file, $local_name);
if ($zip_status) {
$total_file_size += $file_size;
$incremental_file_size += $file_size;
} else {
// Assumption is that we continue?? for some things this would be fatal others it would be ok - leave up to user
DUP_PRO_Log::info("WARNING: Unable to zip file: {$file}");
}
$countFiles++;
$chunk_size_in_bytes = $this->global->ziparchive_chunk_size_in_mb * 1000000;
if ($incremental_file_size > $chunk_size_in_bytes) {
// Only close because of chunk size and file descriptors when in legacy mode
DUP_PRO_Log::trace("closing zip because ziparchive mode = {$this->global->ziparchive_mode} fd count = $used_zip_file_descriptor_count or incremental file size=$incremental_file_size and chunk size = $chunk_size_in_bytes");
$incremental_file_size = 0;
$used_zip_file_descriptor_count = 0;
if ($this->zipArchive->close() == true) {
$adjusted_percent = floor(DUP_PRO_PackageStatus::ARCSTART + ((DUP_PRO_PackageStatus::ARCDONE - DUP_PRO_PackageStatus::ARCSTART) * ($countFiles / (float) $total_file_count)));
$build_progress->next_archive_file_index = $countFiles;
$build_progress->retries = 0;
$this->package->Status = $adjusted_percent;
$this->package->update();
$zip_is_open = false;
DUP_PRO_Log::trace("closed zip");
} else {
$err = 'ZipArchive close failure during file phase using multi-threaded setting.';
$this->setDupArchiveSwitchFix($err);
return $build_progress->failed = true;
}
}
//MT: SERVER THROTTLE
if ($this->throttleDelayInUs !== 0) {
usleep($this->throttleDelayInUs);
}
//MT: MAX WORKER TIME (SECS)
if ($build_progress->timedOut($this->global->php_max_worker_time_in_sec)) {
// Only close because of timeout
$timed_out = true;
$diff = time() - $build_progress->thread_start_time;
DUP_PRO_Log::trace("Timed out after hitting thread time of $diff so quitting zipping early in the file phase");
break;
}
//MT: MAX BUILD TIME (MINUTES)
//Only stop to check on larger files above 100K to avoid checking every single file
if ($file_size > $this->maxBuildTimeFileSize && $this->optMaxBuildTimeOn) {
$elapsed_sec = time() - $this->package->timer_start;
$elapsed_minutes = $elapsed_sec / 60;
if ($elapsed_minutes > $this->global->max_package_runtime_in_min) {
DUP_PRO_Log::trace("ZipArchive: Multi-thread max build time {$this->global->max_package_runtime_in_min} minutes reached killing process.");
return false;
}
}
} else {
$countFiles++;
}
}
DUP_PRO_Log::trace("total file size added to zip = $total_file_size");
if ($zip_is_open) {
DUP_PRO_Log::trace("Doing final zip close after adding $incremental_file_size");
if ($this->zipArchive->close()) {
DUP_PRO_Log::trace("Final zip closed.");
$build_progress->next_archive_file_index = $countFiles;
$build_progress->retries = 0;
$this->package->update();
} else {
$err = __('ZipArchive close failure.', 'duplicator-pro');
$this->setDupArchiveSwitchFix($err);
DUP_PRO_Log::error($err);
return $build_progress->failed = true;
}
}
}
//============================================
//MT: LOG FINAL RESULTS
//============================================
if ($timed_out === false) {
$build_progress->archive_built = true;
$build_progress->retries = 0;
$this->package->update();
$timerAllEnd = DUP_PRO_U::getMicrotime();
$timerAllSum = DUP_PRO_U::elapsedTime($timerAllEnd, $build_progress->archive_start_time);
$zipFileSize = @filesize($zipPath);
DUP_PRO_Log::info("COMPRESSED SIZE: " . DUP_PRO_U::byteSize($zipFileSize));
DUP_PRO_Log::info("ARCHIVE RUNTIME: {$timerAllSum}");
DUP_PRO_Log::info("MEMORY STACK: " . DUP_PRO_Server::getPHPMemory());
if ($this->zipArchive->open() === true) {
$this->package->Archive->file_count = $this->zipArchive->getNumFiles();
$this->package->update();
$this->zipArchive->close();
} else {
DUP_PRO_Log::error("ZipArchive open failure.", "Encountered when retrieving final archive file count.", false);
return $build_progress->failed = true;
}
}
return !$timed_out;
}
/**
* Encodes a UTF8 file and then determines if it is safe to add to an archive
*
* @param string $file The file to test
*
* @return bool Returns true if the file is readable and safe to add to archive
*/
private function isUTF8FileSafe($file)
{
$is_safe = true;
$original_file = $file;
DUP_PRO_Log::trace("[{$file}] is non ASCII");
// Necessary for adfron type files
if (DUP_PRO_STR::hasUTF8($file)) {
$file = utf8_decode($file);
}
if (file_exists($file) === false) {
if (file_exists($original_file) === false) {
DUP_PRO_Log::trace("$file CAN'T BE READ!");
DUP_PRO_Log::info("WARNING: Unable to zip file: {$file}. Cannot be read");
$is_safe = false;
}
}
return $is_safe;
}
/**
* Wrapper for switching to DupArchive quick fix
*
* @param string $message The error message
*
* @return void
*/
private function setDupArchiveSwitchFix($message)
{
$fix_text = __('Click to switch archive engine to DupArchive.', 'duplicator-pro');
$this->setFix(
$message,
$fix_text,
array(
'global' => array(
'archive_build_mode' => DUP_PRO_Archive_Build_Mode::DupArchive,
),
)
);
}
/**
* Sends an error to the trace and build logs and sets the UI message
*
* @param string $message The error message
* @param string $fix The details for how to fix the issue
* @param mixed[] $option The options to set
*
* @return void
*/
private function setFix($message, $fix, $option)
{
DUP_PRO_Log::trace($message);
DUP_PRO_Log::error("$message **FIX: $fix.", '', false);
$system_global = SystemGlobalEntity::getInstance();
$system_global->addQuickFix($message, $fix, $option);
}
}

View File

@@ -0,0 +1,404 @@
<?php
/**
* Build insert iterator
* Standard: PSR-2
*
* @link http://www.php-fig.org/psr/psr-2
*/
defined('ABSPATH') || defined('DUPXABSPATH') || exit;
use Duplicator\Libs\Snap\SnapLog;
/**
* Dump database tables for PHPDump (single and multi)
*/
class DUP_PRO_DB_Build_Iterator implements Iterator
{
const TEMP_COUNTER_FILE_PREFIX = 'dup_pro_db_build_progress_';
const UPDATE_POINTER_FILE_EACH_ROWS = 500;
/** @var ?string store file where pute last offsets */
private $storeProgressFile = null;
/** @var bool is true if use store progress file */
private $isStoreProgress = false;
/** @var bool */
private $isValid = false;
/** @var string[] tables list to iterate */
private $tables = array();
/** @var int count of tables */
private $numTables = 0;
/** @var int current table index */
private $tableIndex = -1;
/** @var int current table offset */
private $tableOffset = 0;
/** @var int table rows */
private $tableRows = 0;
/** @var mixed is last index offset, can be last primary key or unique key, single o compound */
private $lastIndexOffset = 0;
/** @var int total rows insered count. */
private $totalRowsOffset = 0;
/** @var int files size */
private $fileSize = 0;
/** @var bool This value becomes true only by calling a specific function and returns false to the first next row or table. */
private $lastIsCompleteInsert = false;
/** @var callable function called at the beginning of the table parsing. */
private $startTableCallback = null;
/** @var callable function called at the end of the table parsing. */
private $endTableCallback = null;
/** @var int */
private $storePointerRowCount = 0;
/**
* Class constructor
*
* @param string[] $tables tables list to iterate
* @param string|null $storeProgressFile if null the store progress file system isn\'t used. I'ts faster
* @param callable|null $startTableCallback callback called at the begin of current table insert
* @param callable|null $endTableCallback ccallback called at the end of current table insert
*/
public function __construct($tables, $storeProgressFile = null, $startTableCallback = null, $endTableCallback = null)
{
$this->tables = (array) $tables;
$this->numTables = count($this->tables);
if ($this->setStoreProgressFile($storeProgressFile) == false) {
throw new Exception('Can\t set progress file');
}
$this->setPosition();
if (is_callable($startTableCallback)) {
$this->startTableCallback = $startTableCallback;
}
if (is_callable($endTableCallback)) {
$this->endTableCallback = $endTableCallback;
}
}
/**
* set current position if progress file exists or rewrind the iterator
*
* @return void
*/
protected function setPosition()
{
if (!$this->isStoreProgress) {
$this->rewind();
return;
}
DUP_PRO_Log::trace("LOAD DATA DATABASE ITERATOR");
if (($content = file_get_contents($this->storeProgressFile)) === false) {
throw new Exception('Can\'t read database store progress file');
}
if (strlen($content) === 0) {
throw new Exception('Store progress file is empty');
}
if (($data = json_decode($content)) === null) {
throw new Exception('Can\'t decode json progress data content: ' . SnapLog::v2str($content));
}
$this->tableIndex = $data[0];
$this->tableOffset = $data[1];
$this->lastIndexOffset = is_scalar($data[2]) ? $data[2] : (array) $data[2];
$this->totalRowsOffset = $data[3];
$this->lastIsCompleteInsert = $data[4];
$this->tableRows = $data[5];
$this->fileSize = $data[6];
$this->isValid = $data[7];
DUP_PRO_Log::trace("SET POSITION TABLE INDEX " . $this->tableIndex . " OFFSET INDEX " . SnapLog::v2str($this->lastIndexOffset));
}
/**
* save current position in progress file if initialized
*
* @return bool
*/
protected function saveCounterFile()
{
$this->storePointerRowCount = 0;
if (!$this->isStoreProgress) {
return true;
}
$data = array(
$this->tableIndex,
$this->tableOffset,
$this->lastIndexOffset,
$this->totalRowsOffset,
$this->lastIsCompleteInsert,
$this->tableRows,
$this->fileSize,
$this->isValid,
);
if (($dataEncoded = json_encode($data)) === false) {
throw new Exception('Can\'t encode database iterator pointer DATA: ' . SnapLog::v2str($data));
}
// file_put_content is less optimized than fopen,
// fwrite but in some serve keep the hadler file open and do fseek in massive way rarely generate corrupted files.
// So writing and closing the file is the safest method.
if (file_put_contents($this->storeProgressFile, $dataEncoded) !== strlen($dataEncoded)) {
throw new Exception('Can\'t write database store progress file');
}
return true;
}
/**
* rewind current iterator (reset all offset and table counts)
*
* @return void
*/
#[\ReturnTypeWillChange]
public function rewind()
{
DUP_PRO_Log::infoTrace("REWIND DATABASE ITERATOR");
$this->tableIndex = -1;
$this->tableOffset = 0;
$this->lastIndexOffset = 0;
$this->totalRowsOffset = 0;
$this->lastIsCompleteInsert = true;
$this->tableRows = 0;
$this->fileSize = 0;
$this->storePointerRowCount = 0;
$this->next();
}
/**
* remove store progress file
*
* @return void
*/
public function removeCounterFile()
{
if (file_exists($this->storeProgressFile)) {
unlink($this->storeProgressFile);
}
$this->isStoreProgress = false;
$this->storeProgressFile = null;
}
/**
* open store pregress file, if don't exists create and initialize it.
*
* @param string $storeProgressFile path to store progress file
*
* @return boolean
*/
public function setStoreProgressFile($storeProgressFile = null)
{
$this->storeProgressFile = null;
$this->isStoreProgress = false;
if (empty($storeProgressFile)) {
return true;
}
if (($fileExists = file_exists($storeProgressFile))) {
if (!is_writable($storeProgressFile)) {
return false;
}
} elseif (!is_writable(dirname($storeProgressFile))) {
return false;
}
$this->storeProgressFile = $storeProgressFile;
$this->isStoreProgress = true;
if (!$fileExists) {
$this->rewind();
}
return true;
}
/**
* next element (table) of iterator, put all table offsets at 0 and count tableRows
*
* If set call endTableCallback and startTableCallback
*
* @return boolean
*/
#[\ReturnTypeWillChange]
public function next()
{
if ($this->tableIndex >= 0 && is_callable($this->endTableCallback)) {
call_user_func($this->endTableCallback, $this);
}
$this->tableOffset = 0;
$this->lastIndexOffset = 0;
$this->tableRows = 0;
$this->tableIndex++;
if (($this->isValid = ($this->tableIndex < $this->numTables))) {
$res = DUP_PRO_DB::getTablesRows($this->current());
$this->tableRows = $res[$this->current()];
if (is_callable($this->startTableCallback)) {
call_user_func($this->startTableCallback, $this);
}
DUP_PRO_Log::infoTrace("INSERT ROWS TABLE[INDEX:" . $this->tableIndex . "] " . $this->tables[$this->tableIndex] . " NUM ROWS: " . $this->tableRows);
}
$this->saveCounterFile();
return $this->isValid;
}
/**
* increment current table offsets and update store process file if exists
*
* @param mixed $lastIndexOffset last index offset selected, can be a primary key or mixed unique key also composed
* @param int $addFileSize add file size to current file size
*
* @return int return total rows parsed count
*/
public function nextRow($lastIndexOffset = 0, $addFileSize = 0)
{
$this->totalRowsOffset++;
$this->tableOffset++;
$this->lastIndexOffset = $lastIndexOffset;
$this->lastIsCompleteInsert = false;
$this->fileSize += $addFileSize;
$this->storePointerRowCount++;
if ($this->storePointerRowCount >= self::UPDATE_POINTER_FILE_EACH_ROWS) {
$this->saveCounterFile();
}
return $this->totalRowsOffset;
}
/**
* set last is complete inster at true and save it in store preocess file.
*
* @param int $addFileSize size to add
*
* @return void
*/
public function setLastIsCompleteInsert($addFileSize = 0)
{
$this->fileSize += $addFileSize;
$this->lastIsCompleteInsert = true;
$this->saveCounterFile();
}
/**
* @param int $fileSize Size to add
*
* @return void
*/
public function addFileSize($fileSize = 0)
{
$this->fileSize += $fileSize;
$this->saveCounterFile();
}
/**
* @return bool
*/
public function isCurrentTableOffsetValid()
{
return $this->tableOffset < $this->tableRows;
}
/**
*
* @return bool
*/
#[\ReturnTypeWillChange]
public function valid()
{
return $this->isValid;
}
/**
*
* @return string|bool // current table name or false if isn\'t valid
*/
#[\ReturnTypeWillChange]
public function current()
{
return $this->isValid ? $this->tables[$this->tableIndex] : false;
}
/**
*
* @return int // table rows of current table
*/
public function getCurrentRows()
{
return $this->tableRows;
}
/**
*
* @return int // current offset of current table
*/
public function getCurrentOffset()
{
return $this->tableOffset;
}
/**
*
* @return mixed // last index offset selecte, can be a primary key or mixed unique key also composed
*/
public function getLastIndexOffset()
{
return $this->lastIndexOffset;
}
/**
*
* @return int // total rows dumped
*/
public function getTotalsRowsOffset()
{
return $this->totalRowsOffset;
}
/**
*
* @return int // stored file size
*/
public function getFileSize()
{
return $this->fileSize;
}
/**
*
* @return bool // return true if the last inserted sub loop is completed
*/
public function lastIsCompleteInsert()
{
return $this->lastIsCompleteInsert;
}
/**
*
* @return int // current table index
*/
#[\ReturnTypeWillChange]
public function key()
{
return $this->tableIndex;
}
/**
*
* @return int // num table to process
*/
public function count()
{
return $this->numTables;
}
}

View File

@@ -0,0 +1,128 @@
<?php
use Duplicator\Libs\Snap\SnapDB;
class DUP_PRO_Multisite
{
/** @var int[] */
public $FilterSites = array();
/** @var ?string[] */
protected $tablesFilters = null;
/**
* Filter props on json encode
*
* @return string[]
*/
public function __sleep()
{
$props = array_keys(get_object_vars($this));
return array_diff($props, array('tablesFilters'));
}
/**
* Wakeup
*
* @return void
*/
public function __wakeup()
{
if (is_string($this->FilterSites)) {
$this->FilterSites = [];
}
}
/**
* Get dirs to filter
*
* @return string[]
*/
public function getDirsToFilter()
{
if (!empty($this->FilterSites)) {
$path_arr = array();
$wp_content_dir = str_replace("\\", "/", WP_CONTENT_DIR);
foreach ($this->FilterSites as $site_id) {
if ($site_id == 1) {
if (DUP_PRO_MU::getGeneration() == DUP_PRO_MU_Generations::ThreeFivePlus) {
$uploads_dir = $wp_content_dir . '/uploads';
foreach (scandir($uploads_dir) as $node) {
$fullpath = $uploads_dir . '/' . $node;
if ($node == '.' || $node == '.htaccess' || $node == '..') {
continue;
}
if (is_dir($fullpath)) {
if ($node != 'sites') {
$path_arr[] = $fullpath;
}
}
}
} else {
$path_arr[] = $wp_content_dir . '/uploads';
}
} else {
if (file_exists($wp_content_dir . '/uploads/sites/' . $site_id)) {
$path_arr[] = $wp_content_dir . '/uploads/sites/' . $site_id;
}
if (file_exists($wp_content_dir . '/blogs.dir/' . $site_id)) {
$path_arr[] = $wp_content_dir . '/blogs.dir/' . $site_id;
}
}
}
return $path_arr;
} else {
return array();
}
}
/**
* Get tables to filter
*
* @return string[]
*/
public function getTablesToFilter()
{
if (is_null($this->tablesFilters)) {
global $wpdb;
$this->tablesFilters = array();
if (!empty($this->FilterSites)) {
$prefixes = array();
foreach ($this->FilterSites as $site_id) {
$prefix = $wpdb->get_blog_prefix($site_id);
if ($site_id == 1) {
$default_tables = array(
'commentmeta',
'comments',
'links',
//'options', include always options table
'postmeta',
'posts',
'terms',
'term_relationships',
'term_taxonomy',
'termmeta',
);
foreach ($default_tables as $tb) {
$this->tablesFilters[] = $prefix . $tb;
}
} else {
$prefixes[] = $prefix;
}
}
if (count($prefixes)) {
foreach ($prefixes as &$value) {
$value = SnapDB::quoteRegex($value);
}
$regex = '^(' . implode('|', $prefixes) . ').+';
$sql_query = "SHOW TABLES WHERE Tables_in_" . esc_sql(DB_NAME) . " REGEXP '" . esc_sql($regex) . "'";
DUP_PRO_Log::trace('TABLE QUERY PREFIX FILTER: ' . $sql_query);
$sub_tables = $wpdb->get_col($sql_query);
$this->tablesFilters = array_merge($this->tablesFilters, $sub_tables);
}
}
DUP_PRO_Log::traceObject('TABLES TO FILTERS:', $this->tablesFilters);
}
return $this->tablesFilters;
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,601 @@
<?php
/*
Duplicator Pro Plugin
Copyright (C) 2016, Snap Creek LLC
website: snapcreek.com
Duplicator Pro Plugin is distributed under the GNU General Public License, Version 3,
June 2007. Copyright (C) 2007 Free Software Foundation, Inc., 51 Franklin
St, Fifth Floor, Boston, MA 02110, USA
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
defined('ABSPATH') || defined('DUPXABSPATH') || exit;
use Duplicator\Addons\ProBase\License\License;
use Duplicator\Libs\Snap\SnapURL;
use Duplicator\Libs\Snap\SnapUtil;
use Duplicator\Models\Storages\StoragesUtil;
use Duplicator\Models\SystemGlobalEntity;
class DUP_PRO_Package_Runner
{
const DEFAULT_MAX_BUILD_TIME_IN_MIN = 270;
const PACKAGE_STUCK_TIME_IN_SEC = 375; // 75 x 5;
/** @var bool */
public static $delayed_exit_and_kickoff = false;
/**
* Init package runner
*
* @return void
* @throws Exception
*/
public static function init()
{
$kick_off_worker = false;
$global = DUP_PRO_Global_Entity::getInstance();
$system_global = SystemGlobalEntity::getInstance();
if ($global->clientside_kickoff === false) {
if ((time() - $system_global->package_check_ts) < DUP_PRO_Constants::PACKAGE_CHECK_TIME_IN_SEC) {
return;
}
}
if ($global->lock_mode == DUP_PRO_Thread_Lock_Mode::Flock) {
$locking_file = @fopen(DUPLICATOR_PRO_LOCKING_FILE_FILENAME, 'c+');
} else {
$locking_file = true;
}
DUP_PRO_Log::trace('Running package runner init');
if ($locking_file != false) {
if ($global->lock_mode == DUP_PRO_Thread_Lock_Mode::Flock) {
$acquired_lock = (flock($locking_file, LOCK_EX | LOCK_NB) != false);
if ($acquired_lock) {
DUP_PRO_Log::trace("File lock acquired: " . DUPLICATOR_PRO_LOCKING_FILE_FILENAME);
} else {
DUP_PRO_Log::trace("File lock denied " . DUPLICATOR_PRO_LOCKING_FILE_FILENAME);
}
} else {
$acquired_lock = DUP_PRO_U::getSqlLock();
}
if ($acquired_lock) {
DUP_PRO_Log::trace("Acquired lock so executing package runner init core code");
$system_global->package_check_ts = time();
$system_global->save();
$pending_cancellations = DUP_PRO_Package::get_pending_cancellations();
self::cancel_long_running($pending_cancellations);
if (count($pending_cancellations) > 0) {
foreach ($pending_cancellations as $package_id_to_cancel) {
DUP_PRO_Log::trace("looking to cancel $package_id_to_cancel");
$package_to_cancel = DUP_PRO_Package::get_by_id((int) $package_id_to_cancel);
if ($package_to_cancel == false) {
continue;
}
if ($package_to_cancel->Status == DUP_PRO_PackageStatus::STORAGE_PROCESSING) {
$package_to_cancel->cancel_all_uploads();
$package_to_cancel->process_storages();
$package_to_cancel->set_status(DUP_PRO_PackageStatus::STORAGE_CANCELLED);
} else {
$package_to_cancel->set_status(DUP_PRO_PackageStatus::BUILD_CANCELLED);
}
$package_to_cancel->post_scheduled_build_failure();
}
DUP_PRO_Package::clear_pending_cancellations();
}
if (empty($_REQUEST['action']) || $_REQUEST['action'] != 'duplicator_pro_process_worker') {
self::process_schedules();
$kick_off_worker = DUP_PRO_Package::isPackageRunning();
}
if ($global->lock_mode == DUP_PRO_Thread_Lock_Mode::Flock) {
if (!flock($locking_file, LOCK_UN)) {
DUP_PRO_Log::trace("File lock cant release " . $locking_file);
} else {
DUP_PRO_Log::trace("File lock released " . $locking_file);
}
fclose($locking_file);
} else {
DUP_PRO_U::releaseSqlLock();
}
}
} else {
DUP_PRO_Log::trace("Problem opening locking file so auto switching to SQL lock mode");
$global->lock_mode = DUP_PRO_Thread_Lock_Mode::SQL_Lock;
$global->save();
exit();
}
if ($kick_off_worker || self::$delayed_exit_and_kickoff) {
self::kick_off_worker();
} elseif (is_admin() && (isset($_REQUEST['page']) && (strpos($_REQUEST['page'], DUP_PRO_Constants::PLUGIN_SLUG) !== false))) {
DUP_PRO_Log::trace("************kicking off slug worker");
// If it's one of our pages force it to kick off the client
self::kick_off_worker(true);
}
if (self::$delayed_exit_and_kickoff) {
self::$delayed_exit_and_kickoff = false;
exit();
}
}
/**
* Add javascript for cliean side Kick off
*
* @return void
*/
public static function add_kickoff_worker_javascript()
{
$global = DUP_PRO_Global_Entity::getInstance();
$custom_url = strtolower($global->custom_ajax_url);
$CLIENT_CALL_PERIOD_IN_MS = 20000;
// How often client calls into the service
if ($global->ajax_protocol == 'custom') {
if (DUP_PRO_STR::startsWith($custom_url, 'http')) {
$ajax_url = $custom_url;
} else {
// Revert to http standard if they don't have the url correct
$ajax_url = admin_url('admin-ajax.php', 'http');
DUP_PRO_Log::trace("Even though custom ajax url configured, incorrect url set so reverting to $ajax_url");
}
} else {
$ajax_url = admin_url('admin-ajax.php', $global->ajax_protocol);
}
$gateway = array(
'ajaxurl' => $ajax_url,
'client_call_frequency' => $CLIENT_CALL_PERIOD_IN_MS,
'duplicator_pro_process_worker_nonce' => wp_create_nonce('duplicator_pro_process_worker'),
);
wp_register_script('dup-pro-kick', DUPLICATOR_PRO_PLUGIN_URL . 'assets/js/dp-kick.js', array('jquery'), DUPLICATOR_PRO_VERSION);
wp_localize_script('dup-pro-kick', 'dp_gateway', $gateway);
DUP_PRO_Log::trace('KICKOFF: Client Side');
wp_enqueue_script('dup-pro-kick');
}
/**
* Checks active packages for being stuck or running too long and adds them for canceling
*
* @param int[] $pending_cancellations List of package ids to be cancelled
*
* @return void
*/
public static function cancel_long_running(&$pending_cancellations)
{
if (!DUP_PRO_Package::isPackageRunning()) {
return;
}
$active_package = DUP_PRO_Package::get_next_active_package();
if ($active_package === null) {
DUP_PRO_Log::trace("Active package returned null");
return;
}
$global = DUP_PRO_Global_Entity::getInstance();
$system_global = SystemGlobalEntity::getInstance();
$buildStarted = $active_package->timer_start > 0;
$active_package->timer_start = $buildStarted ? $active_package->timer_start : DUP_PRO_U::getMicrotime();
$elapsed_sec = $buildStarted ? DUP_PRO_U::getMicrotime() - $active_package->timer_start : 0;
$elapsed_minutes = $elapsed_sec / 60;
$addedForCancelling = false;
if ($buildStarted && $global->max_package_runtime_in_min > 0 && $elapsed_minutes > $global->max_package_runtime_in_min) {
if ($active_package->build_progress->current_build_mode != DUP_PRO_Archive_Build_Mode::DupArchive) {
$system_global->addQuickFix(
__('Package was cancelled because it exceeded Max Build Time.', 'duplicator-pro'),
sprintf(
__(
'Click button to switch to the DupArchive engine. Please see this %1$sFAQ%2$s for other possible solutions.',
'duplicator-pro'
),
'<a href="' . DUPLICATOR_PRO_DUPLICATOR_DOCS_URL . 'how-to-resolve-schedule-build-failures" target="_blank">',
'</a>'
),
array(
'global' => array(
'archive_build_mode' => DUP_PRO_Archive_Build_Mode::DupArchive,
),
)
);
} elseif ($global->max_package_runtime_in_min < self::DEFAULT_MAX_BUILD_TIME_IN_MIN) {
$system_global->addQuickFix(
__('Package was cancelled because it exceeded Max Build Time.', 'duplicator-pro'),
sprintf(
__(
'Click button to increase Max Build Time. Please see this %1$sFAQ%2$s for other possible solutions.',
'duplicator-pro'
),
'<a href="' . DUPLICATOR_PRO_DUPLICATOR_DOCS_URL . 'how-to-resolve-schedule-build-failures" target="_blank">',
'</a>'
),
array(
'global' => array(
'max_package_runtime_in_min' => self::DEFAULT_MAX_BUILD_TIME_IN_MIN,
),
)
);
}
DUP_PRO_Log::infoTrace("Package $active_package->ID has been going for $elapsed_minutes minutes so cancelling. ($elapsed_sec)");
array_push($pending_cancellations, $active_package->ID);
$addedForCancelling = true;
}
if ((($active_package->Status == DUP_PRO_PackageStatus::AFTER_SCAN) || ($active_package->Status == DUP_PRO_PackageStatus::PRE_PROCESS)) && ($global->clientside_kickoff == false)) {
// Traditionally package considered stuck if > 75 but that was with time % 5 so multiplying by 5 to compensate now
if ($elapsed_sec > self::PACKAGE_STUCK_TIME_IN_SEC) {
DUP_PRO_Log::trace("*** STUCK");
$showDefault = true;
if (isset($_SERVER['AUTH_TYPE']) && $_SERVER['AUTH_TYPE'] == 'Basic' && !$global->basic_auth_enabled) {
$system_global->addQuickFix(
__('Set authentication username and password', 'duplicator-pro'),
__('Automatically set basic auth username and password', 'duplicator-pro'),
array(
'special' => array('set_basic_auth' => 1),
)
);
$showDefault = false;
}
if (SnapURL::isCurrentUrlSSL() && $global->ajax_protocol == 'http') {
$system_global->addQuickFix(
__('Communication to AJAX is blocked.', 'duplicator-pro'),
__('Click button to configure plugin to use HTTPS.', 'duplicator-pro'),
array(
'special' => array('stuck_5percent_pending_fix' => 1),
)
);
} elseif (!SnapURL::isCurrentUrlSSL() && $global->ajax_protocol == 'https') {
$system_global->addQuickFix(
__('Communication to AJAX is blocked.', 'duplicator-pro'),
__('Click button to configure plugin to use HTTP.', 'duplicator-pro'),
array(
'special' => array('stuck_5percent_pending_fix' => 1),
)
);
} elseif ($global->ajax_protocol == 'custom') {
$system_global->addQuickFix(
__('Communication to AJAX is blocked.', 'duplicator-pro'),
__('Click button to fix the admin-ajax URL setting.', 'duplicator-pro'),
array(
'special' => array('stuck_5percent_pending_fix' => 1),
)
);
} elseif ($showDefault) {
$system_global->addTextFix(
__('Communication to AJAX is blocked.', 'duplicator-pro'),
sprintf(
"%s <a href='" . DUPLICATOR_PRO_DUPLICATOR_DOCS_URL . "how-to-resolve-builds-getting-stuck-at-a-certain-point/' target='_blank'>%s</a>",
__('See FAQ:', 'duplicator-pro'),
__('Why is the package build stuck at 5%?', 'duplicator-pro')
)
);
}
DUP_PRO_Log::infoTrace("Package $active_package->ID has been stuck for $elapsed_minutes minutes so cancelling. ($elapsed_sec)");
array_push($pending_cancellations, $active_package->ID);
$addedForCancelling = true;
}
}
if ($addedForCancelling) {
$active_package->buildFail(
'Package was cancelled because it exceeded Max Build Time.',
false
);
} else {
$active_package->save();
}
}
/**
* Kick off worker
*
* @param bool $run_only_if_client If true then only kick off worker if the request came from the client
*
* @return void
*/
public static function kick_off_worker($run_only_if_client = false)
{
/* @var $global DUP_PRO_Global_Entity */
$global = DUP_PRO_Global_Entity::getInstance();
if (!$run_only_if_client || $global->clientside_kickoff) {
$calling_function_name = SnapUtil::getCallingFunctionName();
DUP_PRO_Log::trace("Kicking off worker process as requested by $calling_function_name");
$custom_url = strtolower($global->custom_ajax_url);
if ($global->ajax_protocol == 'custom') {
if (DUP_PRO_STR::startsWith($custom_url, 'http')) {
$ajax_url = $custom_url;
} else {
// Revert to http standard if they don't have the url correct
$ajax_url = admin_url('admin-ajax.php', 'http');
DUP_PRO_Log::trace("Even though custom ajax url configured, incorrect url set so reverting to $ajax_url");
}
} else {
$ajax_url = admin_url('admin-ajax.php', $global->ajax_protocol);
}
DUP_PRO_Log::trace("Attempting to use ajax url $ajax_url");
if ($global->clientside_kickoff) {
add_action('wp_enqueue_scripts', 'DUP_PRO_Package_Runner::add_kickoff_worker_javascript');
add_action('admin_enqueue_scripts', 'DUP_PRO_Package_Runner::add_kickoff_worker_javascript');
} else {
// Server-side kickoff
$ajax_url = SnapURL::appendQueryValue($ajax_url, 'action', 'duplicator_pro_process_worker');
$ajax_url = SnapURL::appendQueryValue($ajax_url, 'now', time());
// $duplicator_pro_process_worker_nonce = wp_create_nonce('duplicator_pro_process_worker');
//require_once(ABSPATH.'wp-includes/pluggable.php');
//$ajax_url = wp_nonce_url($ajax_url, 'duplicator_pro_process_worker', 'nonce');
DUP_PRO_Log::trace('KICKOFF: Server Side');
if ($global->basic_auth_enabled) {
$sglobal = DUP_PRO_Secure_Global_Entity::getInstance();
$args = array(
'blocking' => false,
'headers' => array('Authorization' => 'Basic ' . base64_encode($global->basic_auth_user . ':' . $sglobal->basic_auth_password)),
);
} else {
$args = array('blocking' => false);
}
$args['sslverify'] = false;
wp_remote_get($ajax_url, $args);
}
DUP_PRO_Log::trace("after sent kickoff request");
}
}
/**
* Process schedules by cron
*
* @return void
*/
public static function process()
{
if (!defined('WP_MAX_MEMORY_LIMIT')) {
define('WP_MAX_MEMORY_LIMIT', '512M');
}
if (SnapUtil::isIniValChangeable('memory_limit')) {
@ini_set('memory_limit', WP_MAX_MEMORY_LIMIT);
}
@set_time_limit(7200);
@ignore_user_abort(true);
if (SnapUtil::isIniValChangeable('pcre.backtrack_limit')) {
@ini_set('pcre.backtrack_limit', (string) PHP_INT_MAX);
}
if (SnapUtil::isIniValChangeable('default_socket_timeout')) {
@ini_set('default_socket_timeout', '7200');
// 2 Hours
}
/* @var $global DUP_PRO_Global_Entity */
$global = DUP_PRO_Global_Entity::getInstance();
if ($global->clientside_kickoff) {
DUP_PRO_Log::trace("PROCESS: From client");
session_write_close();
} else {
DUP_PRO_Log::trace("PROCESS: From server");
}
// Only attempt to process schedules if manual isn't running
if ($global->lock_mode == DUP_PRO_Thread_Lock_Mode::Flock) {
$locking_file = fopen(DUPLICATOR_PRO_LOCKING_FILE_FILENAME, 'c+');
} else {
$locking_file = true;
}
if ($locking_file == false) {
DUP_PRO_Log::trace("Problem opening locking file so auto switching to SQL lock mode");
$global->lock_mode = DUP_PRO_Thread_Lock_Mode::SQL_Lock;
$global->save();
exit();
}
// Here we know that $locking_file != false
if ($global->lock_mode == DUP_PRO_Thread_Lock_Mode::Flock) {
$acquired_lock = (flock($locking_file, LOCK_EX | LOCK_NB) != false);
if ($acquired_lock) {
DUP_PRO_Log::trace("File lock acquired " . $locking_file);
} else {
DUP_PRO_Log::trace("File lock denied " . $locking_file);
}
} else {
// DUP_PRO_U::getSqlLock will write details into trace log, logging is not needed here
$acquired_lock = DUP_PRO_U::getSqlLock();
}
if (!$acquired_lock) {
// File locked so another cron already running so just skip
DUP_PRO_Log::trace("File locked so skipping");
return;
}
// Here we know that $acquired_lock == true
self::process_schedules();
$package = DUP_PRO_Package::get_next_active_package();
if ($package != null) {
StoragesUtil::getDefaultStorage()->initStorageDirectory(true);
$dup_tests = self::get_requirements_tests();
if ($dup_tests['Success'] == true) {
$start_time = time();
DUP_PRO_Log::trace("PACKAGE $package->ID:PROCESSING");
ignore_user_abort(true);
if ($package->Status < DUP_PRO_PackageStatus::AFTER_SCAN) {
// Scan step built into package build - used by schedules - NOT manual build where scan is done in web service.
DUP_PRO_Log::trace("PACKAGE $package->ID:SCANNING");
//After scanner runs. Save FilterInfo (unreadable, warnings, globals etc)
$package->create_scan_report();
$package->update();
//del if($package->Archive->ScanStatus == DUP_PRO_Archive::ScanStatusComplete){
$dupe_package = DUP_PRO_Package::get_by_id($package->ID);
$dupe_package->set_status(DUP_PRO_PackageStatus::AFTER_SCAN);
//del }
$end_time = time();
$scan_time = $end_time - $start_time;
//del $end_time = DUP_PRO_U::getMicrotime();
//
// $scan_time = $end_time - $package->Archive->ScanTimeStart;
DUP_PRO_Log::trace("SCAN TIME=$scan_time seconds");
} elseif ($package->Status < DUP_PRO_PackageStatus::COPIEDPACKAGE) {
DUP_PRO_Log::trace("PACKAGE $package->ID:BUILDING");
$package->run_build();
$end_time = time();
$build_time = $end_time - $start_time;
DUP_PRO_Log::trace("BUILD TIME=$build_time seconds");
} elseif ($package->Status < DUP_PRO_PackageStatus::COMPLETE) {
DUP_PRO_Log::trace("PACKAGE $package->ID:STORAGE PROCESSING");
$package->set_status(DUP_PRO_PackageStatus::STORAGE_PROCESSING);
$package->process_storages();
$end_time = time();
$build_time = $end_time - $start_time;
DUP_PRO_Log::trace("STORAGE CHUNK PROCESSING TIME=$build_time seconds");
if ($package->Status == DUP_PRO_PackageStatus::COMPLETE) {
DUP_PRO_Log::trace("PACKAGE $package->ID COMPLETE");
} elseif ($package->Status == DUP_PRO_PackageStatus::ERROR) {
DUP_PRO_Log::trace("PACKAGE $package->ID IN ERROR STATE");
}
$packageCompleteStatuses = array(
DUP_PRO_PackageStatus::COMPLETE,
DUP_PRO_PackageStatus::ERROR,
);
if (in_array($package->Status, $packageCompleteStatuses)) {
$info = "\n";
$info .= "********************************************************************************\n";
$info .= "********************************************************************************\n";
$info .= "DUPLICATOR PRO PACKAGE CREATION OR MANUAL STORAGE TRANSFER END: " . @date("Y-m-d H:i:s") . "\n";
$info .= "NOTICE: Do NOT post to public sites or forums \n";
$info .= "********************************************************************************\n";
$info .= "********************************************************************************\n";
DUP_PRO_Log::infoTrace($info);
}
}
ignore_user_abort(false);
} else {
DUP_PRO_Log::open($package->NameHash);
if ($dup_tests['RES']['INSTALL'] == 'Fail') {
DUP_PRO_Log::info('Installer files still present on site. Remove using Tools > Stored Data > "Remove Installer Files".');
}
DUP_PRO_Log::error(__('Requirements Failed', 'duplicator-pro'), print_r($dup_tests, true), false);
DUP_PRO_Log::traceError('Requirements didn\'t pass so can\'t perform backup!');
$package->post_scheduled_build_failure($dup_tests);
$package->set_status(DUP_PRO_PackageStatus::REQUIREMENTS_FAILED);
}
}
//$kick_off_worker = (DUP_PRO_Package::get_next_active_package() != null);
$kick_off_worker = DUP_PRO_Package::isPackageRunning();
if ($global->lock_mode == DUP_PRO_Thread_Lock_Mode::Flock) {
DUP_PRO_Log::trace("File lock released");
if (!flock($locking_file, LOCK_UN)) {
DUP_PRO_Log::trace("File lock cant release " . $locking_file);
} else {
DUP_PRO_Log::trace("File lock released " . $locking_file);
}
fclose($locking_file);
} else {
DUP_PRO_U::releaseSqlLock();
}
if ($kick_off_worker) {
self::kick_off_worker();
}
}
/**
* Gets the requirements tests
*
* @return array<string,mixed>
*/
public static function get_requirements_tests()
{
$dup_tests = DUP_PRO_Server::getRequirments();
if ($dup_tests['Success'] != true) {
DUP_PRO_Log::traceObject('requirements', $dup_tests);
}
return $dup_tests;
}
/**
* Calculates the earliest schedule run time
*
* @return int
*/
public static function calculate_earliest_schedule_run_time()
{
if (!License::can(License::CAPABILITY_SCHEDULE)) {
return -1;
}
$next_run_time = PHP_INT_MAX;
$schedules = DUP_PRO_Schedule_Entity::get_active();
foreach ($schedules as $schedule) {
if ($schedule->next_run_time == -1) {
$schedule->updateNextRuntime();
}
if ($schedule->next_run_time !== -1 && $schedule->next_run_time < $next_run_time) {
$next_run_time = $schedule->next_run_time;
}
}
if ($next_run_time == PHP_INT_MAX) {
$next_run_time = -1;
}
return $next_run_time;
}
/**
* Start schedule package creation
*
* @return void
*/
public static function process_schedules()
{
// Hack fix - observed issue on a machine where schedule process bombs
$next_run_time = self::calculate_earliest_schedule_run_time();
if ($next_run_time != -1 && ($next_run_time <= time())) {
$schedules = DUP_PRO_Schedule_Entity::get_active();
foreach ($schedules as $schedule) {
$schedule->process();
}
}
}
}

View File

@@ -0,0 +1,355 @@
<?php
use Duplicator\Models\Storages\AbstractStorageEntity;
use Duplicator\Models\Storages\Local\DefaultLocalStorage;
use Duplicator\Models\Storages\StoragesUtil;
use Duplicator\Models\Storages\UnknownStorage;
use VendorDuplicator\Amk\JsonSerialize\JsonSerialize;
defined("ABSPATH") or die("");
abstract class DUP_PRO_Upload_Status
{
const Pending = 0;
const Running = 1;
const Succeeded = 2;
const Failed = 3;
const Cancelled = 4;
}
// Tracks the progress of the package with relation to a specific storage provider
// Used to track a specific upload as well as later report on its' progress
class DUP_PRO_Package_Upload_Info
{
/** @var int<-1,max> */
protected $storage_id = -1;
/** @var int */
public $archive_offset = 0;
/** @var bool Next byte of archive to copy */
public $copied_installer = false;
/** @var bool Whether installer has been copied */
public $copied_archive = false;
/** @var float Whether archive has been copied */
public $progress = 0;
/** @var int 0-100 where this particular storage is at */
public $num_failures = 0;
/** @var bool */
protected $failed = false;
/** @var bool If catastrophic failure has been experienced or num_failures exceeded threshold */
public $cancelled = false;
/** @var scalar */
public $upload_id = null;
/** @var int */
public $failure_count = 0;
/** @var mixed */
public $data = '';
/** @var mixed */
public $data2 = '';
// Storage specific data
// Log related properties - these all SHOULD be public but since we need to json_encode them they have to be public. Ugh.
/** @var bool */
public $has_started = false;
/** @var string */
public $status_message_details = '';
// Details about the storage run (success or failure)
/** @var int */
public $started_timestamp = 0;
/** @var int */
public $stopped_timestamp = 0;
/** @var mixed[] chunk iterator data */
public $chunkPosition = [];
/** @var ?AbstractStorageEntity */
protected $storage = null;
/** @var array<string,mixed> Copy to persistance extra data */
public $copyToExtraData = [];
/**
* Class constructor
*
* @param int $storage_id The storage id
*/
public function __construct($storage_id)
{
$this->setStorageId($storage_id);
}
/**
* Will be called, automatically, when Serialize
*
* @return array<string, mixed>
*/
public function __serialize() // phpcs:ignore PHPCompatibility.FunctionNameRestrictions.NewMagicMethods.__serializeFound
{
$data = JsonSerialize::serializeToData($this, JsonSerialize::JSON_SKIP_MAGIC_METHODS | JsonSerialize::JSON_SKIP_CLASS_NAME);
$data['storage'] = null;
return $data;
}
/**
* Set the storage id
*
* @param int $storage_id The storage id
*
* @return void
*/
public function setStorageId($storage_id)
{
if ($storage_id < 0) {
$this->storage_id = -1;
}
$this->storage_id = (int) $storage_id;
$this->storage = null;
}
/**
* Get the storage object
*
* @return AbstractStorageEntity
*/
public function getStorage()
{
if ($this->storage === null) {
if ($this->storage_id == DefaultLocalStorage::OLD_VIRTUAL_STORAGE_ID) {
// Legacy old packages use virtual storage id -2
$this->storage = StoragesUtil::getDefaultStorage();
$this->storage_id = $this->storage->getId();
} else {
$this->storage = AbstractStorageEntity::getById($this->storage_id);
}
if ($this->storage === false) {
$this->storage = new UnknownStorage();
}
}
return $this->storage;
}
/**
* Get storage id
*
* @return int
*/
public function getStorageId()
{
// For old packages, some storage ids are strings
return (int) $this->storage_id;
}
/**
* Return true if is local
*
* @return bool
*/
public function isLocal()
{
$storage = $this->getStorage();
if ($storage instanceof UnknownStorage) {
return false;
}
return $this->getStorage()->isLocal();
}
/**
* Return true if is remote
*
* @return bool
*/
public function isRemote()
{
$storage = $this->getStorage();
if ($storage instanceof UnknownStorage) {
return false;
}
return !$this->getStorage()->isLocal();
}
/**
* Is failed
*
* @return bool True if upload has failed
*/
public function isFailed()
{
return $this->failed;
}
/**
* Return true if the upload has started
*
* @return bool
*/
public function has_started()
{
return $this->has_started;
}
/**
* Start the upload
*
* @return void
*/
public function start()
{
$this->has_started = true;
$this->started_timestamp = time();
}
/**
* Stop the upload
*
* @return void
*/
public function stop()
{
$this->stopped_timestamp = time();
}
/**
* Get started timestamp
*
* @return int
*/
public function get_started_timestamp()
{
return $this->started_timestamp;
}
/**
* Get stopped timestamp
*
* @return int
*/
public function get_stopped_timestamp()
{
return $this->stopped_timestamp;
}
/**
* Get the status text
*
* @return string
*/
public function get_status_text()
{
$status = $this->get_status();
$status_text = __('Unknown', 'duplicator-pro');
if ($status == DUP_PRO_Upload_Status::Pending) {
$status_text = __('Pending', 'duplicator-pro');
} elseif ($status == DUP_PRO_Upload_Status::Running) {
$status_text = __('Running', 'duplicator-pro');
} elseif ($status == DUP_PRO_Upload_Status::Succeeded) {
$status_text = __('Succeeded', 'duplicator-pro');
} elseif ($status == DUP_PRO_Upload_Status::Failed) {
$status_text = __('Failed', 'duplicator-pro');
} elseif ($status == DUP_PRO_Upload_Status::Cancelled) {
$status_text = __('Cancelled', 'duplicator-pro');
}
return $status_text;
}
/**
* Get the status
*
* @return int
*/
public function get_status()
{
if ($this->cancelled) {
$status = DUP_PRO_Upload_Status::Cancelled;
} elseif ($this->failed) {
$status = DUP_PRO_Upload_Status::Failed;
} elseif ($this->has_started() === false) {
$status = DUP_PRO_Upload_Status::Pending;
} elseif ($this->has_completed(true)) {
$status = DUP_PRO_Upload_Status::Succeeded;
} else {
$status = DUP_PRO_Upload_Status::Running;
}
return $status;
}
/**
* Set the status message details
*
* @param string $status_message_details The status message details
*
* @return void
*/
public function set_status_message_details($status_message_details)
{
$this->status_message_details = $status_message_details;
}
/**
* Get the status message
*
* @return string
*/
public function get_status_message()
{
$message = '';
$status = $this->get_status();
$storage = AbstractStorageEntity::getById($this->storage_id);
if ($storage !== false) {
if ($status == DUP_PRO_Upload_Status::Pending) {
$message = $storage->getPendingText();
} elseif ($status == DUP_PRO_Upload_Status::Failed) {
$message = $storage->getFailedText();
} elseif ($status == DUP_PRO_Upload_Status::Cancelled) {
$message = $storage->getCancelledText();
} elseif ($status == DUP_PRO_Upload_Status::Succeeded) {
$message = $storage->getSuccessText();
} else {
$message = $storage->getActionText();
}
} else {
$message = "Error. Unknown storage id {$this->storage_id}";
DUP_PRO_Log::trace($message);
}
$message_details = $this->status_message_details == '' ? '' : " ($this->status_message_details)";
$message = "$message$message_details";
return $message;
}
/**
* Return true if the upload has completed
*
* @param bool $count_only_success If true then only return true if the upload has completed successfully
*
* @return bool
*/
public function has_completed($count_only_success = false)
{
$retval = false;
if ($count_only_success) {
$retval = (($this->failed == false) && ($this->cancelled == false) && ($this->copied_installer && $this->copied_archive));
} else {
$retval = $this->failed || ($this->copied_installer && $this->copied_archive) || $this->cancelled;
}
if ($retval && ($this->stopped_timestamp == null)) {
// Having to set stopped this way because we aren't OO and allow everyone to set failed/other flags so impossible to know exactly when its done
$this->stop();
}
return $retval;
}
/**
* Increase the failure count
*
* @return void
*/
public function increase_failure_count()
{
$global = DUP_PRO_Global_Entity::getInstance();
$this->failure_count++;
DUP_PRO_Log::infoTrace("Failure count increasing to $this->failure_count [Storage Id: $this->storage_id]");
if ($this->failure_count > $global->max_storage_retries) {
DUP_PRO_Log::infoTrace("* Failure count reached to max level, Storage Status updated to failed [Storage Id: $this->storage_id]");
$this->failed = true;
}
}
}

View File

@@ -0,0 +1,3 @@
<?php
//silent