coabot/src/url_metadata.js

const bot = require('bbot');
const request = require('request');
var request_options = {
  headers: {
    'User-Agent': 'crabot retrieving url metadata (https://wiki.crapaud-fou.org/crabot)'
  }
};



bot.global.text(/(https?:\/\/[^ ,\)"\n]*)/ig, async (b) => {
  // console.log(JSON.stringify(b.match, null, 2));
  for (url of b.match) {
    // console.log(JSON.stringify(url, null, 2));
    if (!/(coa|pad)\.crapaud-fou\.org/.test(url)) {
      request_options.url = url;
      request(request_options, (err, res, body) => {
        // console.log(JSON.stringify(res, null, 2));
        if (!err && res.statusCode === 200) {
          var re = /<title[^>]*>([^<]*)<\/title>/gi;
          var match = re.exec(body);
          if (match && match[1]) {
            var extra = "";
            var link = res.request.uri.href;
            link = link.replace(/[\?&]fbclid=.+/gi, "");
            if (/youtube.com/.test(link)) {
              if (t = /[?&]v=([^&]+)/.exec(link)) {
                link = "https://www.youtube.com/watch?v=" + t[1];
                extra += " - [no-cookie](https://www.youtube-nocookie.com/embed/" + t[1] + ")";
                extra += " - [invidious](https://invidio.us/watch?v=" + t[1] + ")";
              }
            }
            b.respond("[>>>](" + link + ") _" + match[1].replace(/\n/g, " ").trim() + "_" + extra);
          }
        } else {
          console.log('-----------------------------------');
          console.log('URL: ' + url);
          console.log('ERR - - - - - - - - - - - - - - - -');
          console.log(JSON.stringify(err, null, 2));
          console.log('RES - - - - - - - - - - - - - - - -');
          console.log(JSON.stringify(res, null, 2));
          console.log('-----------------------------------');
        }
      });
    }
    await new Promise((resolve) => setTimeout(resolve, 1000));
  }
}, {
  id: 'get-url-metadata'
});