前端之家

  1. 首页
  2. 问答

ftp目录下载触发最大调用堆栈数超出错误 问题建议的解决方案使用队列管理器示例

• 问答

我目前正在使用NodeJS制作备份脚本。该脚本使用FTP / FTPS递归下载目录及其文件和子目录。我正在使用basic-ftp包进行FTP调用。

当我尝试下载包含许多子目录的大目录时，出现Maximum call stack size exceeded错误，但是我找不到原因和发生位置。我看不到任何无限循环或丢失的回叫电话。经过数小时的调试，我再也没有想法了。

我不使用basic-ftp中的downloadDirTo方法，因为我不想在发生错误后停止下载。发生错误时，应继续进行操作，并将错误添加到日志文件中。

存储库在这里：https://github.com/julianpoemp/webspace-backup。

一旦FTPManager准备就绪，我将调用doBackup方法（请参见BackupManager中的方法）。此方法调用FTPManager中定义的downloadFolder方法。

export class BackupManager {

    private ftpManager: FtpManager;

    constructor() {
        osLocale().then((locale) => {
            ConsoleOutput.info(`locale is ${locale}`);
            moment.locale(locale);
        }).catch((error) => {
            ConsoleOutput.error(error);
        });

        this.ftpManager = new FtpManager(AppSettings.settings.backup.root,{
            host: AppSettings.settings.server.host,port: AppSettings.settings.server.port,user: AppSettings.settings.server.user,password: AppSettings.settings.server.password,pasvTimeout: AppSettings.settings.server.pasvTimeout
        });

        this.ftpManager.afterManagerIsReady().then(() => {
            this.doBackup();
        }).catch((error) => {
            ConsoleOutput.error(error);
        });
    }

    public doBackup() {
        let errors = '';
        if (fs.existsSync(path.join(AppSettings.appPath,'errors.log'))) {
            fs.unlinkSync(path.join(AppSettings.appPath,'errors.log'));
        }
        if (fs.existsSync(path.join(AppSettings.appPath,'statistics.txt'))) {
            fs.unlinkSync(path.join(AppSettings.appPath,'statistics.txt'));
        }
        const subscr = this.ftpManager.error.subscribe((message: string) => {
            ConsoleOutput.error(`${moment().format('L LTS')}: ${message}`);
            const line = `${moment().format('L LTS')}:\t${message}\n`;
            errors += line;
            fs.appendFile(path.join(AppSettings.appPath,'errors.log'),line,{
                encoding: 'Utf8'
            },() => {
            });
        });

        let name = AppSettings.settings.backup.root.substring(0,AppSettings.settings.backup.root.lastIndexOf('/'));
        name = name.substring(name.lastIndexOf('/') + 1);
        const downloadPath = (AppSettings.settings.backup.downloadPath === '') ? AppSettings.appPath : AppSettings.settings.backup.downloadPath;

        ConsoleOutput.info(`Remote path: ${AppSettings.settings.backup.root}\nDownload path: ${downloadPath}\n`);

        this.ftpManager.statistics.started = Date.now();
        this.ftpManager.downloadFolder(AppSettings.settings.backup.root,path.join(downloadPath,name)).then(() => {
            this.ftpManager.statistics.ended = Date.now();
            this.ftpManager.statistics.duration = (this.ftpManager.statistics.ended - this.ftpManager.statistics.started) / 1000 / 60;

            ConsoleOutput.success('Backup finished!');
            const statistics = `\n-- Statistics: --
Started: ${moment(this.ftpManager.statistics.started).format('L LTS')}
Ended: ${moment(this.ftpManager.statistics.ended).format('L LTS')}
Duration: ${this.ftpManager.getTimeString(this.ftpManager.statistics.duration * 60 * 1000)} (H:m:s)

Folders: ${this.ftpManager.statistics.folders}
Files: ${this.ftpManager.statistics.files}
Errors: ${errors.split('\n').length - 1}`;

            ConsoleOutput.log('\n' + statistics);
            fs.writeFileSync(path.join(AppSettings.appPath,'statistics.txt'),statistics,{
                encoding: 'utf-8'
            });
            if (errors !== '') {
                ConsoleOutput.error(`There are errors. Please read the errors.log file for further information.`);
            }
            subscr.unsubscribe();
            this.ftpManager.close();
        }).catch((error) => {
            ConsoleOutput.error(error);
            this.ftpManager.close();
        });
    }
}

import * as ftp from 'basic-ftp';
import {FileInfo} from 'basic-ftp';
import * as Path from 'path';
import * as fs from 'fs';
import {Subject} from 'rxjs';
import {FtpEntry,FTPFolder} from './ftp-entry';
import {ConsoleOutput} from './ConsoleOutput';
import moment = require('moment');

export class FtpManager {
    private isReady = false;
    private _client: ftp.Client;
    private currentDirectory = '';

    public readyChange: Subject<boolean>;
    public error: Subject<string>;
    private connectionOptions: FTPConnectionOptions;

    public statistics = {
        folders: 0,files: 0,started: 0,ended: 0,duration: 0
    };

    private recursives = 0;

    constructor(path: string,options: FTPConnectionOptions) {
        this._client = new ftp.Client();
        this._client.ftp.verbose = false;
        this.readyChange = new Subject<boolean>();
        this.error = new Subject<string>();
        this.currentDirectory = path;
        this.connectionOptions = options;


        this.connect().then(() => {
            this.isReady = true;
            this.gotTo(path).then(() => {
                this.onready();
            }).catch((error) => {
                ConsoleOutput.error('ERROR: ' + error);
                this.onConnectionFailed();
            });
        });
    }

    private connect(): Promise<void> {
        return new Promise<void>((resolve,reject) => {
            this._client.access({
                host: this.connectionOptions.host,user: this.connectionOptions.user,password: this.connectionOptions.password,secure: true
            }).then(() => {
                resolve();
            }).catch((error) => {
                reject(error);
            });
        });
    }

    private onready = () => {
        this.isReady = true;
        this.readyChange.next(true);
    };

    private onConnectionFailed() {
        this.isReady = false;
        this.readyChange.next(false);
    }

    public close() {
        this._client.close();
    }

    public async gotTo(path: string) {
        return new Promise<void>((resolve,reject) => {
            if (this.isReady) {
                ConsoleOutput.info(`open ${path}`);
                this._client.cd(path).then(() => {
                    this._client.pwd().then((dir) => {
                        this.currentDirectory = dir;
                        resolve();
                    }).catch((error) => {
                        reject(error);
                    });
                }).catch((error) => {
                    reject(error);
                });
            } else {
                reject(`FTPManager is not ready. gotTo ${path}`);
            }
        });
    }

    public async listEntries(path: string): Promise<FileInfo[]> {
        if (this.isReady) {
            return this._client.list(path);
        } else {
            throw new Error('FtpManager is not ready. list entries');
        }
    }

    public afterManagerIsReady(): Promise<void> {
        return new Promise<void>((resolve,reject) => {
            if (this.isReady) {
                resolve();
            } else {
                this.readyChange.subscribe(() => {
                        resolve();
                    },(error) => {
                        reject(error);
                    },() => {
                    });
            }
        });
    }

    public async downloadFolder(remotePath: string,downloadPath: string) {
        this.recursives++;

        if (this.recursives % 100 === 99) {
            ConsoleOutput.info('WAIT');
            await this.wait(0);
        }

        if (!fs.existsSync(downloadPath)) {
            fs.mkdirSync(downloadPath);
        }

        try {
            const list = await this.listEntries(remotePath);
            for (const fileInfo of list) {
                if (fileInfo.isDirectory) {
                    const folderPath = remotePath + fileInfo.name + '/';
                    try {
                        await this.downloadFolder(folderPath,Path.join(downloadPath,fileInfo.name));
                        this.statistics.folders++;
                        ConsoleOutput.success(`${this.getcurrentTimeString()}===> Directory downloaded: ${remotePath}\n`);
                    } catch (e) {
                        this.error.next(e);
                    }
                } else if (fileInfo.isFile) {
                    try {
                        const filePath = remotePath + fileInfo.name;
                        if (this.recursives % 100 === 99) {
                            ConsoleOutput.info('WAIT');
                            await this.wait(0);
                        }
                        await this.downloadFile(filePath,downloadPath,fileInfo);
                    } catch (e) {
                        this.error.next(e);
                    }
                }
            }
            return true;
        } catch (e) {
            this.error.next(e);
            return true;
        }
    }

    public async downloadFile(path: string,downloadPath: string,fileInfo: FileInfo) {
        this.recursives++;
        if (fs.existsSync(downloadPath)) {
            const handler = (info) => {
                let procent = Math.round((info.bytes / fileInfo.size) * 10000) / 100;
                if (isnaN(procent)) {
                    procent = 0;
                }
                let procentStr = '';
                if (procent < 10) {
                    procentStr = '__';
                } else if (procent < 100) {
                    procentStr = '_';
                }
                procentStr += procent.toFixed(2);

                ConsoleOutput.log(`${this.getcurrentTimeString()}---> ${info.type} (${procentStr}%): ${info.name}`);
            };

            if (this._client.closed) {
                try {
                    await this.connect();
                } catch (e) {
                    throw new Error(e);
                }
            }
            this._client.trackProgress(handler);
            try {
                await this._client.downloadTo(Path.join(downloadPath,fileInfo.name),path);
                this._client.trackProgress(undefined);
                this.statistics.files++;
                return true;
            } catch (e) {
                throw new Error(e);
            }
        } else {
            throw new Error('downloadPath does not exist');
        }
    }

    public chmod(path: string,permission: string): Promise<void> {
        return new Promise<void>((resolve,reject) => {
            this._client.send(`SITE CHMOD ${permission} ${path}`).then(() => {
                console.log(`changed chmod of ${path} to ${permission}`);
                resolve();
            }).catch((error) => {
                reject(error);
            });
        });
    }

    public getcurrentTimeString(): string {
        const duration = Date.now() - this.statistics.started;
        return moment().format('L LTS') + ' | Duration: ' + this.getTimeString(duration) + ' ';
    }

    public getTimeString(timespan: number) {
        if (timespan < 0) {
            timespan = 0;
        }

        let result = '';
        const minutes: string = this.formatNumber(this.getMinutes(timespan),2);
        const seconds: string = this.formatNumber(this.getSeconds(timespan),2);
        const hours: string = this.formatNumber(this.getHours(timespan),2);

        result += hours + ':' + minutes + ':' + seconds;

        return result;
    }

    private formatNumber = (num,length): string => {
        let result = '' + num.toFixed(0);
        while (result.length < length) {
            result = '0' + result;
        }
        return result;
    };

    private getSeconds(timespan: number): number {
        return Math.floor(timespan / 1000) % 60;
    }

    private getMinutes(timespan: number): number {
        return Math.floor(timespan / 1000 / 60) % 60;
    }

    private getHours(timespan: number): number {
        return Math.floor(timespan / 1000 / 60 / 60);
    }

    public async wait(time: number): Promise<void> {
        return new Promise<void>((resolve) => {
            setTimeout(() => {
                resolve();
            },time);
        });
    }
}


export interface FTPConnectionOptions {
    host: string;
    port: number;
    user: string;
    password: string;
    pasvTimeout: number;
}
ris2000 回答：ftp目录下载触发最大调用堆栈数超出错误 问题建议的解决方案使用队列管理器示例

问题

在FtpManager.downloadFolder函数中，我看到使用downloadFolder递归调用相同的await方法。您的Maximum call stack exceeded错误可能来自此，因为您的首次调用将需要在遍历所有子目录时将所有内容保留在内存中。

建议的解决方案

您可以使用以下算法来设置队列系统，而不是await进行递归操作：

  • 将当前文件夹添加到队列中
  • 该队列不为空：
    • 获取队列中的第一个文件夹（并将其删除）
    • 列出其中的所有条目
    • 下载所有文件
    • 将所有子文件夹添加到队列中

这使您可以循环下载许多文件夹，而不必使用递归。每次循环迭代将独立运行，这意味着根目录下载的结果将不依赖于其中的deeeeeep文件树。

使用队列管理器

有很多NodeJS的队列管理器模块，它们允许您进行并发，超时等操作。我过去使用的一个简单地命名为queue。它具有许多有用的功能，但是需要更多的工作才能在您的项目中实施。因此，对于这个答案，我没有使用外部队列模块，因此您可以看到其背后的逻辑。随时搜索queue，job，concurrency ...

示例

我想直接在您自己的代码中实现该逻辑，但是我不使用Typescript，所以我想我会做一个简单的文件夹复制函数，它使用相同的逻辑。

注意：为简单起见，我没有添加任何错误处理，这只是概念证明！您可以找到一个使用此here on my Github的演示项目。

这是我的操作方式：

const fs = require('fs-extra');
const Path = require('path');

class CopyManager {
  constructor() {
    // Create a queue accessible by all methods
    this.folderQueue = [];
  }

  /**
   * Copies a directory
   * @param {String} remotePath
   * @param {String} downloadPath
   */
  async copyFolder(remotePath,downloadPath) {
    // Add the folder to the queue
    this.folderQueue.push({ remotePath,downloadPath });
    // While the queue contains folders to download
    while (this.folderQueue.length > 0) {
      // Download them
      const { remotePath,downloadPath } = this.folderQueue.shift();
      console.log(`Copy directory: ${remotePath} to ${downloadPath}`);
      await this._copyFolderAux(remotePath,downloadPath);
    }
  }

  /**
   * Private internal method which copies the files from a folder,* but if it finds subfolders,simply adds them to the folderQueue
   * @param {String} remotePath
   * @param {String} downloadPath
   */
  async _copyFolderAux(remotePath,downloadPath) {
    await fs.mkdir(downloadPath);
    const list = await this.listEntries(remotePath);
    for (const fileInfo of list) {
      if (fileInfo.isDirectory) {
        const folderPath = Path.join(remotePath,fileInfo.name);
        const targetPath = Path.join(downloadPath,fileInfo.name);
        // Push the folder to the queue
        this.folderQueue.push({ remotePath: folderPath,downloadPath: targetPath });
      } else if (fileInfo.isFile) {
        const filePath = Path.join(remotePath,fileInfo.name);
        await this.copyFile(filePath,downloadPath,fileInfo);
      }
    }
  }

  /**
   * Copies a file
   * @param {String} filePath
   * @param {String} downloadPath
   * @param {Object} fileInfo
   */
  async copyFile(filePath,fileInfo) {
    const targetPath = Path.join(downloadPath,fileInfo.name);
    console.log(`Copy file: ${filePath} to ${targetPath}`);
    return await fs.copy(filePath,targetPath);
  }

  /**
   * Lists entries from a folder
   * @param {String} remotePath
   */
  async listEntries(remotePath) {
    const fileNames = await fs.readdir(remotePath);
    return Promise.all(
      fileNames.map(async name => {
        const stats = await fs.lstat(Path.join(remotePath,name));
        return {
          name,isDirectory: stats.isDirectory(),isFile: stats.isFile()
        };
      })
    );
  }
}

module.exports = CopyManager;
,

我找到了问题的根源。发出最大调用堆栈超出错误的是pkg软件包：www.github.com/zeit/pkg/issues/681。

当我直接使用Windows上的节点对其进行测试时，它可以正常工作。我将降级到Node 10或寻找其他解决方案。

感谢@blex的帮助！

ftpjavascriptnode.js
本文链接：https://www.f2er.com/3169819.html

大家都在问