将TensorflowJS模型保存到Firebase存储的最佳实践?

似乎有可能进行基于HTTP请求的“发布”。

https://www.tensorflow.org/js/guide/save_load

https://firebase.google.com/docs/functions/http-events

担心我可能会使其复杂化,是否有一种简单的方法可以在客户端js中创建json文件(也许也避免保存到本地存储中?),然后使用put函数简单地上传到存储中? https://firebase.google.com/docs/storage/web/upload-files

在这些领域经验不足,找不到针对此场景的特定内容。

调查:

似乎这是使用云功能 Write image file to Firebase Storage from HTTP function

更新

当前正在研究使用基于内置Tensorflowjs HTTP请求的保存(具有权重的bin文件和模型结构json文件),该保存将由云功能处理,而不是确定这是最好的方法还是行得通,但到目前为止我还是最好的。现在还在研究如何通过http请求发送文件夹名称?

lgq890116 回答:将TensorflowJS模型保存到Firebase存储的最佳实践?

我的客户端Tensorflow JS请求通过HTTP请求保存到云存储。也许有更好的方法可以做到这一点,但这至少对我有用。

var api = 'https://us-central1-your-uniquelocation.cloudfunctions.net/uploadModel';
var res = await model.save(tf.io.browserHTTPRequest(api,{method: 'POST',headers:{'Authorization':'test','Content-Type': 'multipart/form-data; boundary=----WebKitFormBoundaryVoB3gVGBQHG0btQR'}}));
console.log(res);

然后将 index.js用于我的云函数

const functions = require('firebase-functions');
var admin = require("firebase-admin");
var serviceAccount = require("./service-account-credentials.json");

admin.initializeApp({
    credential: admin.credential.cert(serviceAccount),//databaseURL: "https://yoururl.firebaseio.com",storageBucket: "yoururl.appspot.com"
});

var bucket = admin.storage().bucket("yoururl.appspot.com");

const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');


exports.uploadModel = functions.https.onRequest((req,res) => {
    res.set('Access-Control-Allow-Origin','*');
    res.set('Access-Control-Allow-Credentials','true');
    if (req.method === 'OPTIONS') {
        // Send response to OPTIONS requests
        console.log("send response 204"); 

        res.set('Access-Control-Allow-Methods','GET');
        res.set('Access-Control-Allow-Headers','Authorization');
        res.set('Access-Control-Max-Age','3600');

        res.status(204).send();
     }

    console.log("metho--->"+req.method);
    console.log(req.headers);

    const busboy = new Busboy({headers: req.headers});
    const tmpdir = os.tmpdir();

    // This object will accumulate all the fields,keyed by their name
    const fields = {};

    // This object will accumulate all the uploaded files,keyed by their name.
    const uploads = {};

    // This code will process each non-file field in the form.
    busboy.on('field',(fieldname,val) => {
        // TODO(developer): Process submitted field values here
        console.log(`Processed field ${fieldname}: ${val}.`);
        fields[fieldname] = val;
    });

    const fileWrites = [];
    //res.status(200).send();

    // This code will process each file uploaded.
    busboy.on('file',file,filename) => {
        // Note: os.tmpdir() points to an in-memory file system on GCF
        // Thus,any files in it must fit in the instance's memory.
        console.log(`Processed file ${filename}`);
        const filepath = path.join(tmpdir,filename);
        uploads[fieldname] = filepath;

        const writeStream = fs.createWriteStream(filepath);
        file.pipe(writeStream);

        // File was processed by Busboy; wait for it to be written to disk.
        const promise = new Promise((resolve,reject) => {
            file.on('end',() => {
                writeStream.end();
            });
            writeStream.on('finish',resolve);
            writeStream.on('error',reject);
        });
        fileWrites.push(promise);
    });

    // Triggered once all uploaded files are processed by Busboy.
    // We still need to wait for the disk writes (saves) to complete.
    busboy.on('finish',() => {
        Promise.all(fileWrites).then(() => {
        // TODO(developer): Process saved files here
            for (const name in uploads) {
                const file = uploads[name];
                bucket.upload(file).then(()=>{
                    fs.unlinkSync(file);
                    console.log("saved");
                });
            }
            res.send();
        });
    });
    busboy.end(req.rawBody);
});

更新:

增加了通过http请求发送文件夹名称的功能,这也许再一次不完美,但是可以正常工作,最初很难弄清楚。

针对我的云功能的新 index.js ,适用于那些希望通过http请求发送自定义文件夹名称的人。

const functions = require('firebase-functions');
var admin = require("firebase-admin");
var serviceAccount = require("./service-account-credentials.json");

admin.initializeApp({
    credential: admin.credential.cert(serviceAccount),storageBucket: "YOURURL.appspot.com"
});

var bucket = admin.storage().bucket();

const path = require('path');
const os = require('os');
const fs = require('fs');
const Busboy = require('busboy');

exports.uploadModel2s = functions.https.onRequest((req,'true');
    if (req.method === 'OPTIONS'){
        console.log("send response 204"); 
        res.set('Access-Control-Allow-Methods','Authorization,ModelId');
        res.set('Access-Control-Max-Age','3600');
        return res.status(204).send();
    }

    console.log("method ---> "+req.method);
    const modelid = req.headers['modelid'];
    console.log("modelid ---> "+modelid);
    console.log(req.headers);

    const busboy = new Busboy({headers: req.headers});
    const tmpdir = os.tmpdir();

    const fields = {};
    const uploads = {};

    busboy.on('field',val) => {
        console.log(`Processed field ${fieldname}: ${val}.`);
        fields[fieldname] = val;
    });

    const fileWrites = [];

    busboy.on('file',filename) => {
        console.log(`Processed 2 file ${modelid} ${filename}`);
        const filepath = path.join(tmpdir,filename);
        uploads[fieldname] = filepath;
        console.log('PATH: '+filepath);
        const writeStream = fs.createWriteStream(filepath);
        file.pipe(writeStream);

        const promise = new Promise((resolve,reject);
        });
        fileWrites.push(promise);
    });

    busboy.on('finish',() => {
        Promise.all(fileWrites).then(() => {
            for (const name in uploads){
                const file = uploads[name];
                const destpath = "/m_"+modelid+"/"+name;
                console.log("SAVE ME HERE: "+destpath);
                bucket.upload(file,{ destination: destpath,public: true }).then(() => {
                    if(name == "model.weights.bin"){
                        fs.unlinkSync(file);
                        console.log("save weights");
                        res.send();
                    }else{
                        console.log("save model");
                    }
                });
            }
        });
    });
    busboy.end(req.rawBody);
});

新客户代码

var api = 'https://yourproject.cloudfunctions.net/uploadModel2s';
var res = await model.save(tf.io.browserHTTPRequest(api,'ModelId':m}}));

安全风险

为了加快开发速度,我在上传时使用了 public:true ,因此我可以轻松访问文件进行下载,但这绝对不安全。

本文链接:https://www.f2er.com/3056514.html

大家都在问