remove robots.txt endpoint

This reverts #833 because the contributor did not respond to our
request to sign the CLA

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

internal/httpd/httpd.go

@@ -95,7 +95,6 @@ const (
 	rolesPath                             = "/api/v2/roles"
 	ipListsPath                           = "/api/v2/iplists"
 	healthzPath                           = "/healthz"
-	robotsTxtPath                         = "/robots.txt"
 	webRootPathDefault                    = "/"
 	webBasePathDefault                    = "/web"
 	webBasePathAdminDefault               = "/web/admin"