#!ruby -w
# frozen_string_literal: true

require 'wlonspider/version'
require 'wlonspider/spider_config'
require 'wlonspider/downloader'
require 'wlonspider/page'
require 'wlonspider/spider'
require 'wlonspider/download_link'
require 'wlonspider/process_status'
require 'optparse'
require 'logger'
require 'nokogiri'
require 'cgi'
require 'uri'
# require 'profile'

module Wlonspider
  LOG = Logger.new(STDOUT)
  LOG.level = Logger::INFO
  LOG.datetime_format = '%H:%M:%S'
  class Dispatcher
    attr_reader :start_url, :thread_count

    @@interrupted = false
    @@links = {}
    @@mutexadd = Mutex.new
    @@mutexget = Mutex.new

    def initialize(*args)
      @thread_count = args[0].nil? ? Wlonspider::Config::SpiderConfig.download_thread : args[0]
      @start_url = args[1].nil? ? Wlonspider::Config::SpiderConfig.start_url : CGI.escape(args[1]).gsub('%23', '#').gsub('%3A', ':').gsub('%2F', '/').gsub('%3F', '?').gsub('%3D', '=').gsub('%26', '&')
      Wlonspider::Config::SpiderConfig.fetch_level = args[1].nil? ? Wlonspider::Config::SpiderConfig.fetch_level : 1

      #   ActiveRecord::Base.establish_connection(
      #     :adapter  => "mysql2",
      #     :host     => "localhost",
      #     :username => "dol",
      #     :password => "dol",
      #     :database => "dol",
      #     :pool=> 10
      # )
      # 设置ActiveRecord运行log的指向
      ActiveRecord::Base.logger = LOG
    end

    def start
      Wlonspider::Dispatcher.add_a_url(start_url, 1)
      threads = (1..thread_count).map do |i|
        Thread.new(i) do |thread_index|
          Wlonspider::Spider.new(thread_index).crawl
        end
      end
    rescue Exception => e
      Wlonspider::LOG.error("Dispatcher::start #{e}")
      Wlonspider::LOG.error("Dispatcher::start #{e.backtrace.join("\n")}")
    ensure
      threads.each(&:join)
    end

    def self.interrupted=(val)
      @@interrupted = val
    end

    def self.interrupted
      @@interrupted
    end

    def self.add_a_url(url, level, *args)
      @@mutexadd.synchronize do
        url.sub!('//', 'https://') if url.start_with?('//')
        uri = URI.parse(url)
        unless @@links.include?(uri.request_uri)
          @@links[uri.request_uri] = Wlonspider::DownloadLink.new(url, uri, level, args)
          Wlonspider::LOG.debug("Dispatcher::add_a_url #{url}")
        end
        uri
      end
    rescue Exception => e
      Wlonspider::LOG.error("Dispatcher::add_a_url #{url}")
      Wlonspider::LOG.error("Dispatcher::add_a_url #{e}")
      Wlonspider::LOG.error(e.backtrace.join("\n").to_s)
    end

    def self.get_a_url
      @@mutexget.synchronize do
        urls = @@links.select do |_k, v|
          v.process_status < Wlonspider::ProcessStatus::PROCESS
        end
        if !urls.nil? && !urls.empty?
          urls.min { |x, y| y[1].priority <=> x[1].priority }[1].process_status = Wlonspider::ProcessStatus::PROCESS unless urls.nil?
          url = urls.min { |x, y| y[1].priority <=> x[1].priority }[1]
          Wlonspider::LOG.info("Dispatcher#get_a_url #{CGI.unescape(url.raw_url)}| Total: #{@@links.length} |Process: #{@@links.select { |_k, v| v.process_status == Wlonspider::ProcessStatus::PROCESS }.length}| Rest: #{@@links.select { |_k, v| v.process_status < Wlonspider::ProcessStatus::PROCESS }.length} | Finished:#{@@links.select { |_k, v| v.process_status == Wlonspider::ProcessStatus::DOWNLOADED }.length} ")
          return url
        end
      end
    end

    def self.links_marshal
      unless File.exist? '../db/links.yml'
        Wlonspider::LOG.info('dump links')
        File.open('../db/links.yml', 'w') do |file|
          file.puts YAML.dump(@@links)
        end
      end
    end
  end
end

trap('INT') do
  Wlonspider::Dispatcher.links_marshal
  Wlonspider::Dispatcher.interrupted = true
  puts 'Exit spider......'
end

options = {}
option_parser = OptionParser.new do |opts|
  # 这里是这个命令行工具的帮助信息
  opts.banner = 'Dol spider command line help info'
  opts.separator  ''
  opts.separator  'Commands'
  opts.separator  '     start: start spider'
  opts.separator  '     stop:  stop spider'
  opts.separator  ''
  opts.separator  'Start options'

  # 下面第一项是Short option（没有可以直接在引号间留空），第二项是Long option，第三项是对Option的描述
  # 下面的“value”就是用户使用时输入的argument
  opts.on('-p', '--page Url', 'fetch a single page with the given url') do |value|
    # 这个部分就是使用这个Option后执行的代码
    options[:single_url] = value
  end

  # Option 作为flag，带argument，用于将argument作为数值解析，比如"name"信息
  # 下面的“value”就是用户使用时输入的argument
  opts.on('-t', '--thread N', Numeric, 'downloader thread number') do |value|
    options[:thread_count] = value
  end
end.parse!

case ARGV[0]
when 'start'
  puts 'Start spider'
  if options[:thread_count]
    Wlonspider::Dispatcher.new(options[:thread_count]).start
  elsif options[:single_url]
    Wlonspider::Dispatcher.new(options[:thread_count], options[:single_url]).start
  else
    Wlonspider::Dispatcher.new.start
  end

when 'stop'
  puts "call stop on options #{options.inspect}"
end
