#filldupedb-Script v2.01

var sites_to_skip = <none>
var skip_dirs = *NUKED*
var with_archive=true


#days to backfill from paramstr or take 5 as default
var tmp=$paramstr(1)
if $tmp=
  var tmp=5
end

#check if i can encode date from it...if not its alwready a date i guess
var end_date=ENCODEDATE $tmp
if $end_date =
  var end_date=$tmp
end

######################################################################################

writeln Begin of DupeDB-filling started (oldest date: $end_date)...


var source_sites = ACTIVESITELIST

for sname in $source_sites

  if $sname = $sites_to_skip
    goto site_done
  end


  if $with_archive=true
    var sections=RULEARCHIVESECTIONS $sname
  end
  if $with_archive!=true
    var sections=RULESECTIONS $sname
  end

  for sec in $sections

    var dir_count=0
    var parsed_pathes=
    var i=0
    :backfill_start

    var source_dirs=$sname.sectionpath($sec,$i)
    if $with_archive=true
      var archive=$sname.archivepath($sec,$i)
      if $archive!=
        if $source_dirs!=
          var source_dirs=$source_dirs;
        end
        var source_dirs=$source_dirs$archive
      end
    end
    var sdir_temp=$sname.sectionpath($sec,500)
    if $with_archive=true
      var archive=$sname.archivepath($sec,500)
      if $archive!=
        if $sdir_temp!=
          var sdir_temp=$sdir_temp;
        end
        var sdir_temp=$sdir_temp$archive
      end
    end

    for sdir in $source_dirs

      #did we have this one parsed already?
      if $sdir = $parsed_pathes
        #writeln [-] already parsed $sdir ($parsed_pathes)
        goto dir_done
      end

      if $parsed_pathes<>
        var parsed_pathes=$parsed_pathes|
      end
      var parsed_pathes=$parsed_pathes$sdir
          

      #create check-dir
      var dir_count=$dir_count+1
      var sdir_chk=GETPART $sdir_temp $dir_count ";"

      #check if its a dated dir or not....
      var dated_dir=1
      if $sdir_chk=$sdir
        var dated_dir=0
      end

      #connect + cwd + refresh
      select $sname
      if $sname.connected = false
        connect $sname
      end
      if $sname.connected = false
        goto dir_done
      end

      cwd $sdir
      if $sname.ok = false
        goto dir_done
      end

      refresh
      select ALL
      deselect MASK $skip_dirs
      if $dated_dir = 0
        limitselection mask $end_date
      end

      var que=SELECTION

      var beauty_i=$i+1
      writeln [$sec] Site: %bold$sname:%bold$sdir, Dated: $dated_dir, Day-X: $beauty_i
          
      for item in $que
        add_to_db $sname$sdir/$item
      end
      flush_db

      :dir_done

    end

    if $source_dirs!=$sdir_temp
      var dated_dir=1
    end

    if $dated_dir=1
      var i=$i+1
      var tmp=ENCODEDATE $i
      if $tmp != $end_date
        goto backfill_start
      end
    end
    :section_done
    
  end
  :site_done  

end