Experimental browser for the Atmosphere
Websites need to push back against those crawlers. Idk, put a malicious file that will crash the crawler in a directory that human users won't access and that is forbidden by robots.txt? That should only hit bots that deserve it.
May 4, 2025, 3:58 PM
{ "uri": "at://did:plc:5ydouar2zcb4zp3cly2i2xxt/app.bsky.feed.post/3loe7tydrfk2f", "cid": "bafyreifxpmc7chiq4m3mmwjcmj2xyhsbge5umm2facs25byq7p5ywyrx7e", "value": { "text": "Websites need to push back against those crawlers. Idk, put a malicious file that will crash the crawler in a directory that human users won't access and that is forbidden by robots.txt? That should only hit bots that deserve it.", "$type": "app.bsky.feed.post", "langs": [ "en" ], "reply": { "root": { "cid": "bafyreibtk2yli34bvm5xoykirddy5s74amwfqhqgfi46maksx2ljd3ckcq", "uri": "at://did:plc:lxid5bd2zxkrgx47uy74txsy/app.bsky.feed.post/3llt2a6b3bu2v" }, "parent": { "cid": "bafyreibtk2yli34bvm5xoykirddy5s74amwfqhqgfi46maksx2ljd3ckcq", "uri": "at://did:plc:lxid5bd2zxkrgx47uy74txsy/app.bsky.feed.post/3llt2a6b3bu2v" } }, "createdAt": "2025-05-04T15:58:21.639Z" } }