infinitered/nsfwjs

NSFWjs not working in lambda

Open

#521 aperta il 21 giu 2021

Vedi su GitHub
 (5 commenti) (0 reazioni) (0 assegnatari)JavaScript (507 fork)batch import
Node.jsUpstreambugenhancementhelp wanted

Metriche repository

Star
 (7090 star)
Metriche merge PR
 (Nessuna PR mergiata in 30 g)

Descrizione

I tried using nsfwjs to detect images upload to s3 as safe or not but it is not working in lambda, I tried increasing memory to 10GB. But it still timed out :(

Log:

START RequestId: 18f3c1c7-3341-4973-b35c-b638cadd851e Version: $LATEST
2021-06-21T03:34:53.247Z    18f3c1c7-3341-4973-b35c-b638cadd851e    WARN
============================
Hi there :wave:. Looks like you are running TensorFlow.js in Node.js. To speed things up dramatically, install our node backend, which binds to TensorFlow C++, by running npm i @tensorflow/tfjs-node, or npm i @tensorflow/tfjs-node-gpu if you have CUDA. Then call require('@tensorflow/tfjs-node'); (-gpu suffix for CUDA) at the start of your program. Visit https://github.com/tensorflow/tfjs-node for more details.
============================
END RequestId: 18f3c1c7-3341-4973-b35c-b638cadd851e
REPORT RequestId: 18f3c1c7-3341-4973-b35c-b638cadd851e    Duration: 30033.57 ms    Billed Duration: 30000 ms    Memory Size: 10240 MB    Max Memory Used: 121 MB    Init Duration: 606.54 ms
2021-06-21T03:35:23.277Z 18f3c1c7-3341-4973-b35c-b638cadd851e Task timed out after 30.03 seconds

Code (for local testing):

//  const aws = require('aws-sdk');
const tf = require('@tensorflow/tfjs');
const sizeOf = require('image-size');
const jpeg = require('jpeg-js');
const nsfwjs = require('nsfwjs')
const axios = require('axios')


//const s3 = new aws.S3({ apiVersion: '2006-03-01' });



//exports.handler = async (event, context) => {
async function abc(){
  const model = await nsfwjs.load();

  const readImage = buf => {
    var imgprodata = sizeOf(buf)
    if(imgprodata.type != "JPEG" && imgprodata.type != "JPG" && imgprodata.type != "jpeg"&& imgprodata.type != "jpg"){
      let opt = {
        height: imgprodata.height,
        width: imgprodata.width,
        data: buf,
      }
      const pixels = jpeg.encode(opt,100)
      return pixels
    } else {
    const pixels = jpeg.decode(buf, true)
    return pixels
    }
  }
  
  const imageByteArray = (image, numChannels) => {
    const pixels = image.data
    const numPixels = image.width * image.height;
    const values = new Int32Array(numPixels * numChannels);
  
    for (let i = 0; i < numPixels; i++) {
      for (let channel = 0; channel < numChannels; ++channel) {
        values[i * numChannels + channel] = pixels[i * 4 + channel];
      }
    }
  
    return values
  }
  
  const imageToInput = (image, numChannels) => {
    const values = imageByteArray(image, numChannels)
    const outShape = [image.height, image.width, numChannels];
    const input = tf.tensor3d(values, outShape, 'int32');

  
    return input;
  }

  
  // const key = event.Records[0].s3.object.key;
  const key = '7c1e158e-26a8-44a1-b219-bf85741e4754'

  var pic = await axios.get(`https://public-usercontent.dogegram.xyz/storage/${key}`, {
    responseType: 'arraybuffer'
  });

  var pix = await readImage(pic.data);
  var img = await imageToInput(pix, 3)

  
  const predictions = await model.classify(img)



  console.log(predictions)

  return 'done'
  
}
//}


abc()

For testing:

npm init -y
npm i @tensorflow/tfjs image-size jpeg-js nsfwjs axios

Guida contributor