add new URLs to robots.txt to prevent crawling
tree38cccd0d622750a448ca0a2943a7b7d94619d06e
-rw-r--r-- 708 .gitignore
-rw-r--r-- 1501 LICENSE
-rw-r--r-- 931 README
drwxr-xr-x - cmd
-rw-r--r-- 297 dcs-batch-common.service
-rw-r--r-- 387 dcs-common.service
-rw-r--r-- 227 dcs-index-backend.service
-rw-r--r-- 238 dcs-index-backend@.service
-rw-r--r-- 360 dcs-package-importer.service
-rw-r--r-- 249 dcs-source-backend.service
-rw-r--r-- 541 dcs-update-ranking.service
-rw-r--r-- 127 dcs-update-ranking.timer
-rw-r--r-- 321 dcs-varz-to-influxdb@.service
-rw-r--r-- 365 dcs-web.service
drwxr-xr-x - debian
drwxr-xr-x - howto
drwxr-xr-x - index
-rw-r--r-- 3442 nginx.example
drwxr-xr-x - rackspace
drwxr-xr-x - ranking
drwxr-xr-x - regexp
-rw-r--r-- 108 schema.sql
drwxr-xr-x - shardmapping
drwxr-xr-x - sshutil
drwxr-xr-x - static
-rwxr-xr-x 219 swap-index.sh
drwxr-xr-x - testdata
-rwxr-xr-x 1467 update-index.sh
drwxr-xr-x - varz
-rwxr-xr-x 2139 verify-new-files.sh