community-repo/working-groups/distributors/archive/giza/migration/verify.js

const fs = require("fs");
const axios = require("axios");

const oldFilename = `oldChannels.json`;
const newFilename = `newVideos.json`;
const resultsFilename = `migration_result.json`;
const { channels, videos } = require("./ids.js");
console.log(
  `Loaded IDs of ${videos.length} transferred videos in ${channels.length} channels.`
);
//const { channels } = require("./ru.js");
// channels.map((c) => c.id).reduce((id=> axios.head(`${}`))

const oldChannels = `query { channels (limit: 10000) { id createdAt updatedAt createdInBlock
    title category { name } ownerMember {id handle } isCensored isPublic
    coverPhotoAvailability coverPhotoDataObject { joystreamContentId size }
    avatarPhotoAvailability avatarPhotoDataObject { joystreamContentId size }
    videos { id title mediaDataObject {size} mediaAvailability thumbnailPhotoAvailability }
}}`;

const queryVideos = `query { videos (limit:10000) { id channelId language{iso} media { id } }}`;

const queryVideo = (id) => `query { videos(where: {id_eq: "${id}"})  {
  id channelId language{iso} media { id } }}`;

//curl 'https://ipfs.joystreamstats.live/graphql' -H 'Accept-Encoding: gzip, deflate, br' -H 'Content-Type: application/json' -H 'Accept: application/json' -H 'Connection: keep-alive' -H 'DNT: 1' -H 'Origin: https://ipfs.joystreamstats.live' --data-binary '{"query":"query { videos { id channelId language{iso} media { id } }}"}' --compressed

const getOldChannels = () =>
  axios
    .post(`https://hydra-sumer.joystream.org/graphql`, { query: oldChannels })
    .then(({ data }) => {
      fs.writeFileSync(oldFilename, JSON.stringify(data.data.channels));
      return data.data.channels;
    })
    .catch(({ message, response }) => {
      console.log(response.status, message, response.data);
      return [];
    });

const printResults = (results) => {
  const available = results.filter((v) => v.status === `available`);
  console.log(`${available.length} of ${results.length} assets are available.`);
  const notok = results.filter((v) => v.status !== `available`);
  if (!notok.length) return;

  // print table
  const header = `\n### Created Videos without media file\n\n${notok.length} have no content.\n\n| Channel | Video | Asset | Status |\n|---|---|---|---|\n`;
  const row = ({ id, channelId, media, status }) =>
    `| ${channelId} | ${id} | ${media?.id || ``} | ${status || ``} |`;
  const sorted = notok.sort((a, b) => a.channelId - b.channelId);
  console.log(header + sorted.map((video) => row(video)).join(`\n`));
};

const testVideos = async (list, oldResults) => {
  if (!list?.length) return console.log(`empty result from QN`);
  console.log(`Testing availability of ${list.length} videos (~5..15min)`);
  let results = oldResults || [];
  const timer = setInterval(
    () => console.log(`Tested ${results.length} videos.`),
    5000
  );
  for (const video of list) {
    const { id, channelId, language, media } = video;
    if (!videos.find((map) => +map[1] === +id)) continue; // not migrated
    if (!media) {
      console.log(`Skipping video ${id}: no media info.`);
      results.push({ ...video, status: `no media` });
      continue;
    }
    const status = await axios
      .head(`https://storage-1.joystream.org/argus/api/v1/assets/${media.id}`)
      .then(({ status, statusText }) =>
        status === 200 ? `available` : `${status} ${statusText}`
      )
      .catch((error) => error.message);
    if (status !== "available") console.log(`Not available:`, video);
    results.push({ ...video, status });
  }
  fs.writeFileSync(resultsFilename, JSON.stringify(results));
  console.log(`Wrote ${resultsFilename}.`);
  printResults(results);
  clearInterval(timer);
};

const countSize = (ids, channels) => {
  let acceptedVideos = 0;
  let channelsSize = 0;
  let sizes = [];
  let notacceptedbuttransferred = [];
  channels
    .filter((c) => ids.find((id) => id[0] === +c.id))
    .map((channel) => {
      if (channel.isCensored) console.log(`channel ${cid} was censored!`);
      if (channel.coverPhotoDataObject)
        channelsSize += channel.coverPhotoDataObject.size;
      if (channel.avatarPhotoDataObject)
        channelsSize += channel.avatarPhotoDataObject.size;
      channel.videos.forEach((v) => {
        if (v.mediaAvailability === "ACCEPTED")
          sizes.push(v.mediaDataObject.size);
        else notacceptedbuttransferred.push(v);
      });
    });
  const bytes = sizes.reduce((a, b) => a + b, 0);
  console.log(
    `${sizes.length} videos were transferred ${bytes} bytes + ${channelsSize} avatars + covers.`
  );
  if (notacceptedbuttransferred) {
    const lines = notacceptedbuttransferred.map(
      (v) =>
        `| ${v.id} | ${v.title} | ${v.mediaDataObject.size} | ${v.mediaAvailability} | ${v.thumbnailPhotoAvailability} |`
    );
    console.log(
      `\n### Migrated ${lines.length} videos with empty source\n\n|ID|Title|Size|Upload|Thumbnail|\n|---|---|---|---|---|\n`,
      lines.join(`\n`)
    );
  }
  return bytes + channelsSize;
};

// load old channels from disk or old QN
fs.stat(oldFilename, async (err, stat) => {
  const oldChannels = !err
    ? JSON.parse(fs.readFileSync(oldFilename, "utf-8"))
    : await getOldChannels();
  console.log(`Loaded info of ${oldChannels.length} old channels.`);
  const size = countSize(channels, oldChannels);

  // find censored
  const censored = oldChannels.filter((c) => c.isCensored);
  if (!censored.length) return;
  console.log(
    `\n### Censored source channels\n\nFound ${censored.length} censored channels in migration set:\n`,
    "```\n" + censored.map((j) => JSON.stringify(j)).join("\n") + "\n```"
  );
});

// load new videos from disk or new QN
fs.stat(newFilename, async (err, stat) => {
  const newVideos = !err
    ? JSON.parse(fs.readFileSync(newFilename, "utf-8"))
    : await axios
        .post(`https://ipfs.joystreamstats.live/graphql`, {
          query: queryVideos,
        })
        .then(({ data }) => {
          fs.writeFileSync(newFilename, JSON.stringify(data.data.videos));
          return data.data.videos;
        })
        .catch((error) => {
          console.log(error.message, error);
          return [];
        });

  // find migrated of all new videos
  const migrated = newVideos.filter((v) =>
    videos.find((ids) => ids[1] === +v.id)
  );
  if (!migrated.length)
    return console.log(
      `Something went wrong. No migrated videos found. Contact your administrator.`
    );
  console.log(`Loaded info of ${migrated.length} migrated videos.`);
  fs.stat(resultsFilename, async (err, stat) => {
    if (err) return testVideos(migrated);
    console.log(`Loading old results.`);
    const results = JSON.parse(fs.readFileSync(resultsFilename, "utf-8"));
    const nottested = videos.filter(
      (i) => !results.find((r) => +r.id === i[1])
    );
    console.log(nottested.length, `not tested before.`);
    if (nottested.length) return testVideos(nottested, results);
    console.log(`Nothing to do. all migrated Videos were tested.`);
    printResults(results);
  });
});