eb3e58da91
This looks like a very sane default robots.txt. We can modify it as required. Change-Id: I8b9d3aa63538388e319f0216535f7a1d977f4885
36 lines
755 B
Plaintext
36 lines
755 B
Plaintext
# This was kindly seeded with a mix of
|
|
# https://git.lelux.fi/theel0ja/gitea-robots.txt/src/branch/master/robots.txt
|
|
# and
|
|
# https://github.com/robots.txt
|
|
# at 2020-07-01
|
|
#
|
|
# Some commented out items are left to indicate we have considered
|
|
# them and would like to explicitly allow them for indexing while they
|
|
# are not causing problems.
|
|
|
|
User-agent: *
|
|
|
|
# Disallow: /avatars
|
|
# Disallow: /user/*
|
|
# Disallow: /*/*/src/commit/*
|
|
# Disallow: /*/*/commit/*
|
|
|
|
Disallow: /*/*/activity/*
|
|
Disallow: /vendor/librejs.html
|
|
Disallow: /api/swagger
|
|
Disallow: /swagger.*.json
|
|
|
|
# Language spam
|
|
Disallow: /*?lang=
|
|
|
|
# From github
|
|
Disallow: */archive/
|
|
Disallow: */blame/
|
|
# Disallow: /raw/*
|
|
Disallow: /.git/
|
|
Disallow: */.git/
|
|
Disallow: /*.git$
|
|
Disallow: /*q=
|
|
|
|
Crawl-delay: 2
|