Experimental browser for the Atmosphere
🐞Web data wars & exclusivity: More websites are restricting AI crawlers with robots.txt, ToS, lawsuits, and other anti-crawling measures. AI developers frequently circumvent these restrictions or negotiate exclusive deals for key data, dividing up access on the web. 6/
Apr 9, 2025, 3:25 PM
{ "uri": "at://did:plc:evvussoazdkvsld475dfbuci/app.bsky.feed.post/3lmfcdypzmd2p", "cid": "bafyreihq5jsp7eyml6eykxvnczqpe427hqyse53nnpnhlxmje4byhqedc4", "value": { "text": "🐞Web data wars & exclusivity: More websites are restricting AI crawlers with robots.txt, ToS, lawsuits, and other anti-crawling measures. AI developers frequently circumvent these restrictions or negotiate exclusive deals for key data, dividing up access on the web.\n\n6/", "$type": "app.bsky.feed.post", "langs": [ "en" ], "reply": { "root": { "cid": "bafyreigkmph43tjdkwakwfxe6oin6kgi562zunkpnrfb73fnjfk53zlxwu", "uri": "at://did:plc:evvussoazdkvsld475dfbuci/app.bsky.feed.post/3lmfcdxsuyk2p" }, "parent": { "cid": "bafyreiczap5p5finlr4z6xv2kfqofxwfgbcb7zbgvtxpugjxufxjfi5z4m", "uri": "at://did:plc:evvussoazdkvsld475dfbuci/app.bsky.feed.post/3lmfcdypyn32p" } }, "createdAt": "2025-04-09T15:25:21.901Z" } }