build(js): Add robots.txt to dev previews (#43472)

static/robots-dev.txt

@@ -0,0 +1,3 @@
+# In development and development previews disallow all bots from indexing
+User-agent: *
+Disallow: /

webpack.config.ts

@@ -392,10 +392,21 @@ const appConfig: Configuration = {
     new CopyPlugin({
       patterns: [
         {
-          from: `${staticPrefix}/images/logo-sentry.svg`,
+          from: path.join(staticPrefix, 'images/logo-sentry.svg'),
           to: 'entrypoints/logo-sentry.svg',
           toType: 'file',
         },
+        // Add robots.txt when deploying in preview mode so public previews do
+        // not get indexed by bots.
+        ...(IS_DEPLOY_PREVIEW
+          ? [
+              {
+                from: path.join(staticPrefix, 'robots-dev.txt'),
+                to: 'robots.txt',
+                toType: 'file' as const,
+              },
+            ]
+          : []),
       ],
     }),
   ],