'use strict';

const Uniqueue = require('uniqueue');

const { push, pop, remaining, queued, processed, clear } = new Uniqueue();

const urls = [
  'https://google.com',
  'https://google.com',
  'https://google.com/',
  'https://www.google.com'
];

// Only add unique items to the queue
urls.forEach(url => {
  push(url);
});
console.log('=>', queued.size, processed.size, remaining());
// => 3 0 3

// Extract items from queue
while (remaining() > 0) {
  console.log('=> remaining:', remaining());
  console.log('=>', pop());
  console.log('=> processed:', processed.size);
  console.log();
}
// => remaining: 3
// => https://google.com
// => processed: 1
//
// => remaining: 2
// => https://google.com/
// => processed: 2
//
// => remaining: 1
// => https://www.google.com
// => processed: 3

// Clear the queue
clear();
console.log('=>', queued.size, processed.size, remaining());
// => 0 0 0

// Use a custom matcher function to prevent similar duplicates
const matcher = (a, b) => {
  return (
    a.replace(/[^a-z]/gi, '') === b.replace(/[^a-z]/gi, '') ||
    a.replace(/:\/\/www\./i, '://') === b.replace(/:\/\/www\./i, '://')
  );
};
urls.forEach(url => {
  push(url, matcher);
});
console.log('=>', [...queued]);
// => [ 'https://google.com' ]

// * Use a custom matcher function during initialization
const queue = new Uniqueue(matcher);
// ...