diff --git a/httpd/httpd.go b/httpd/httpd.go index 864f9e4d..218f40cd 100644 --- a/httpd/httpd.go +++ b/httpd/httpd.go @@ -86,6 +86,7 @@ const ( providerEventsPath = "/api/v2/events/provider" sharesPath = "/api/v2/shares" healthzPath = "/healthz" + robotsTxtPath = "/robots.txt" webRootPathDefault = "/" webBasePathDefault = "/web" webBasePathAdminDefault = "/web/admin" diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 885c4fef..b20f04ab 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -118,6 +118,7 @@ const ( providerEventsPath = "/api/v2/events/provider" sharesPath = "/api/v2/shares" healthzPath = "/healthz" + robotsTxtPath = "/robots.txt" webBasePath = "/web" webBasePathAdmin = "/web/admin" webAdminSetupPath = "/web/admin/setup" @@ -8910,6 +8911,13 @@ func TestHealthCheck(t *testing.T) { assert.Equal(t, "ok", rr.Body.String()) } +func TestRobotsTxtCheck(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, "/robots.txt", nil) + rr := executeRequest(req) + checkResponseCode(t, http.StatusOK, rr) + assert.Equal(t, "User-agent: *\nDisallow: /", rr.Body.String()) +} + func TestGetWebRootMock(t *testing.T) { req, _ := http.NewRequest(http.MethodGet, "/", nil) rr := executeRequest(req) diff --git a/httpd/server.go b/httpd/server.go index 187edec1..c428e866 100644 --- a/httpd/server.go +++ b/httpd/server.go @@ -1137,6 +1137,10 @@ func (s *httpdServer) initializeRouter() { render.PlainText(w, r, "ok") }) + s.router.Get(robotsTxtPath, func(w http.ResponseWriter, r *http.Request) { + render.PlainText(w, r, "User-agent: *\nDisallow: /") + }) + // share API exposed to external users s.router.Get(sharesPath+"/{id}", s.downloadFromShare) s.router.Post(sharesPath+"/{id}", s.uploadFilesToShare)