0:{"root":[["$","meta","1",{"property":"csp-nonce","nonce":"766c4e44-27d9-4991-a5f5-1b4f3552645f"}],"$L1"],"formState":"$undefined","returnValue":"$undefined"}
2:"$Sreact.fragment"
3:I["7ada5cab5be8",[],"Header",1]
4:"$Sreact.suspense"
:HL["/assets/_layout-BzYdn0f6.css","stylesheet"]
1:["$","$2","2",{"children":[[["$","link","css:/assets/_layout-BzYdn0f6.css",{"rel":"stylesheet","precedence":"vite-rsc/importer-resources","href":"/assets/_layout-BzYdn0f6.css","data-rsc-css-href":"/assets/_layout-BzYdn0f6.css"}],"$undefined"],["$","html",null,{"lang":"en","className":"$undefined","children":[["$","head",null,{"children":[["$","meta",null,{"charSet":"UTF-8"}],["$","meta",null,{"name":"viewport","content":"width=device-width, initial-scale=1.0"}],["$","link",null,{"rel":"icon","type":"image/png","href":"/static/icons/apple-icon.png"}],["$","link",null,{"rel":"icon","type":"image/png","sizes":"32x32","href":"/static/icons/favicon-32x32.png"}],["$","link",null,{"rel":"apple-touch-icon","href":"/static/icons/apple-icon.png"}],["$","meta",null,{"name":"theme-color","content":"#1e40af"}],["$","meta",null,{"name":"language","content":"en_US"}],["$","meta",null,{"httpEquiv":"content-language","content":"en_US"}]]}],["$","body",null,{"className":"min-h-screen flex flex-col","children":[["$","$L3",null,{}],["$","main",null,{"className":"main-content grow bg-gray-100","children":["$","$4",null,{"fallback":[[["$","link","css:/assets/_layout-BzYdn0f6.css",{"rel":"stylesheet","precedence":"vite-rsc/importer-resources","href":"/assets/_layout-BzYdn0f6.css","data-rsc-css-href":"/assets/_layout-BzYdn0f6.css"}],"$undefined"],["$","div",null,{"className":"flex justify-center items-center p-8","children":["$","div",null,{"className":"animate-spin rounded-full h-8 w-8 border-b-2 border-indigo-600"}]}]],"children":[[["$","title",null,{"children":"Further thoughts on hindering screenscraping | Gleez"}],["$","meta",null,{"name":"description","content":"Further thoughts on hindering screen-scraping We previously listed some means to try to stop screen-scraping, but since it is an ongoing topic for us, it…"}],["$","meta",null,{"name":"keywords","content":"further, thoughts, hindering, screenscraping, Gleez, privacy, secure productivity"}],["$","meta",null,{"name":"author","content":"Sunny"}],["$","meta",null,{"property":"og:title","content":"Further thoughts on hindering screenscraping | Gleez"}],["$","meta",null,{"property":"og:description","content":"Further thoughts on hindering screen-scraping We previously listed some means to try to stop screen-scraping, but since it is an ongoing topic for us, it…"}],["$","meta",null,{"property":"og:type","content":"article"}],["$","meta",null,{"property":"og:url","content":"https://gleez.com/s/further-thoughts-on-hindering-screen-scraping"}],["$","meta",null,{"property":"og:image","content":"/static/og.png"}],["$","meta",null,{"property":"og:image:alt","content":"Further thoughts on hindering screenscraping"}],["$","meta",null,{"property":"og:site_name","content":"Gleez"}],["$","meta",null,{"property":"og:locale","content":"en_US"}],["$","meta",null,{"property":"article:published_time","content":"2010-01-06T09:00:50.000Z"}],["$","meta",null,{"property":"article:modified_time","content":"2018-12-31T17:59:34.000Z"}],["$","meta",null,{"property":"article:author","content":"https://gleez.com/author/73326181482902"}],[],["$","meta",null,{"name":"twitter:card","content":"summary_large_image"}],["$","meta",null,{"name":"twitter:title","content":"Further thoughts on hindering screenscraping | Gleez"}],["$","meta",null,{"name":"twitter:description","content":"Further thoughts on hindering screen-scraping We previously listed some means to try to stop screen-scraping, but since it is an ongoing topic for us, it…"}],["$","meta",null,{"name":"twitter:image","content":"/static/og.png"}],["$","meta",null,{"name":"twitter:image:alt","content":"Further thoughts on hindering screenscraping"}],["$","meta",null,{"name":"twitter:site","content":"@gleeztech"}],["$","meta",null,{"name":"twitter:creator","content":"@gleeztech"}],["$","link",null,{"rel":"canonical","href":"https://gleez.com/s/further-thoughts-on-hindering-screen-scraping"}],["$","meta",null,{"name":"robots","content":"index, follow"}],["$","meta",null,{"name":"googlebot","content":"index, follow, max-snippet:-1, max-image-preview:large"}],["$","script",null,{"type":"application/ld+json","dangerouslySetInnerHTML":{"__html":"{\"@context\":\"https://schema.org\",\"@type\":\"BlogPosting\",\"headline\":\"Further thoughts on hindering screenscraping\",\"description\":\"Further thoughts on hindering screen-scraping We previously listed some means to try to stop screen-scraping, but since it is an ongoing topic for us, it…\",\"image\":\"/static/og.png\",\"author\":{\"@type\":\"Person\",\"name\":\"Sunny\"},\"publisher\":{\"@type\":\"Organization\",\"name\":\"Gleez\",\"logo\":{\"@type\":\"ImageObject\",\"url\":\"https://gleez.tech/static/logo-512.png\"}},\"datePublished\":\"2010-01-06T09:00:50.000Z\",\"dateModified\":\"2018-12-31T17:59:34.000Z\",\"mainEntityOfPage\":{\"@type\":\"WebPage\",\"@id\":\"https://gleez.com/s/further-thoughts-on-hindering-screen-scraping\"}}"}}]],"$L5"]}]}],"$L6"]}]]}]]}]
7:T40f,data:image/svg+xml;utf8,%3Csvg%20width%3D%2232%22%20height%3D%2232%22%20viewBox%3D%220%200%2032%2032%22%20%0A%20%20%20%20%20%20%20%20%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20role%3D%22img%22%20aria-label%3D%22Avatar%20for%20Sunny%22%3E%0A%20%20%20%20%20%20%3Crect%20width%3D%22100%25%22%20height%3D%22100%25%22%20fill%3D%22hsl(151%2C%2071%25%2C%2051%25)%22%20rx%3D%223.2%22%20%2F%3E%0A%20%20%20%20%20%20%3Ctext%20x%3D%2250%25%22%20y%3D%2250%25%22%20%0A%20%20%20%20%20%20%20%20%20%20%20%20font-family%3D%22sans-serif%22%20%0A%20%20%20%20%20%20%20%20%20%20%20%20font-size%3D%2212.8%22%20%0A%20%20%20%20%20%20%20%20%20%20%20%20font-weight%3D%22bold%22%20%0A%20%20%20%20%20%20%20%20%20%20%20%20fill%3D%22%23FFFFFF%22%20%0A%20%20%20%20%20%20%20%20%20%20%20%20text-anchor%3D%22middle%22%20%0A%20%20%20%20%20%20%20%20%20%20%20%20dominant-baseline%3D%22central%22%0A%20%20%20%20%20%20%20%20%20%20%20%20style%3D%22user-select%3A%20none%3B%22%3E%0A%20%20%20%20%20%20%20%20S%0A%20%20%20%20%20%20%3C%2Ftext%3E%0A%20%20%20%20%3C%2Fsvg%3E8:Te39,Further thoughts on hindering screen-scraping
We previously listed some means to try to stop screen-scraping, but since it is an ongoing topic for us, it bears revisiting. Any site can be scraped, but some require such an influx of time and resources as to make it prohibitively expensive. Some of the common methods to do so are:
Turing tests
The most common implementation of the Turning Test is the old CAPTCHA that tries to ensure a human reads the text in an image, and feeds it into a form.
We have found a large number of sites that implement a very weak CAPTCHA that takes only a few minutes to get around. On the other hand, there are some very good implementations of Turing Tests that we would opt not to deal with given the choice, but a sophisticated OCR can sometimes overcome those, or many bulletin board spammers have some clever tricks to get past these.
Data as images
Sometimes you know which parts of your data are valuable. In that case it becomes reasonable to replace such text with an image. As with the Turing Test, there is ORC software that can read it, and there’s no reason we can’t save the image and have someone read it later.
Often times, however, listing data as an image without a text alternate is in violation of the Americans with Disabilities Act (ADA), and can be overcome with a couple of phone calls to a company’s legal department.
Code obfuscation
Using something like a JavaScript function to show data on the page though it’s not anywhere in the HTML source is a good trick. Other examples include putting prolific, extraneous comments through the page or having an interactive page that orders things in an unpredictable way (and the example I think of used CSS to make the display the same no matter the arrangement of the code.)
CSS Sprites
Recently we’ve encountered some instances where a page has one images containing numbers and letters, and used CSS to display only the characters they desired. This is in effect a combination of the previous 2 methods. First we have to get that master-image and read what characters are there, then we’d need to read the CSS in the site and determine to what character each tag was pointing.
While this is very clever, I suspect this too would run afoul the ADA, though I’ve not tested that yet.
Limit search results
Most of the data we want to get at is behind some sort of form. Some are easy, and submitting a blank form will yield all of the results. Some need an asterisk or percent put in the form. The hardest ones are those that will give you only so many results per query. Sometimes we just make a loop that will submit the letters of the alphabet to the form, but if that’s too general, we must make a loop to submit all combination of 2 or 3 letters–that’s 17,576 page requests.
IP Filtering
On occasion, a diligent webmaster will notice a large number of page requests coming from a particular IP address, and block requests from that domain. There are a number of methods to pass requests through alternate domains, however, so this method isn’t generally very effective.
Site Tinkering
Scraping always keys off of certain things in the HTML. Some sites have the resources to constantly tweak their HTML so that any scrapes are constantly out of date. Therefore it becomes cost ineffective to continually update the scrape for the constantly changing conditions.
5:["$","$4",null,{"fallback":[[["$","link","css:/assets/_layout-BzYdn0f6.css",{"rel":"stylesheet","precedence":"vite-rsc/importer-resources","href":"/assets/_layout-BzYdn0f6.css","data-rsc-css-href":"/assets/_layout-BzYdn0f6.css"}],"$undefined"],["$","div",null,{"className":"flex justify-center items-center p-8","children":["$","div",null,{"className":"animate-spin rounded-full h-8 w-8 border-b-2 border-indigo-600"}]}]],"children":["$","div",null,{"className":"bg-white/95 mt-0.5 py-6 sm:py-12","children":["$","article",null,{"className":"mx-auto w-full max-w-2xl format format-sm sm:format-base lg:format-lg format-blue dark:format-invert","children":[["$","header",null,{"className":"mb-4 lg:mb-6 not-format","children":[["$","h1",null,{"className":"mb-4 text-3xl font-extrabold leading-tight text-gray-900 lg:mb-6 lg:text-4xl dark:text-white","children":"Further thoughts on hindering screen-scraping"}],["$","address",null,{"className":"flex items-center mb-6 not-italic","children":["$","div",null,{"className":"inline-flex items-center text-sm text-gray-900 dark:text-white","children":[["$","img",null,{"className":"mr-3 w-8 h-8 rounded-full","src":"$7","alt":"Sunny"}],["$","a",null,{"href":"/author/sunny_73326181482902","rel":"author","className":"font-bold capitalize hover:underline dark:text-white","children":"Sunny"}],["$","span",null,{"className":"mx-1 text-md text-gray-400 dark:text-gray-500","children":"·"}],["$","span",null,{"className":"inline-flex items-center ml-1 text-gray-500 dark:text-gray-400","children":"3 min read"}],["$","span",null,{"className":"mx-1 text-md text-gray-400 dark:text-gray-500","children":"·"}],["$","time",null,{"dateTime":"Wed Jan 06 2010","className":"text-gray-500 dark:text-gray-400","children":"Jan 6, 2010"}],[["$","span",null,{"className":"mx-1 text-md text-gray-400 dark:text-gray-500","children":"·"}],["$","a","tips-and-tricks",{"href":"/t/tips-and-tricks","className":"capitalize text-sm text-gray-500 dark:text-gray-400 hover:text-gray-600","children":"tips & tricks"}]]]}]}]]}],["$","div",null,{"className":"lead prose mt-4 text-lg leading-8 text-gray-600","children":["$","div",null,{"dangerouslySetInnerHTML":{"__html":"$8"}}]}],"$L9","$La"]}]}]}]
b:T63b,M12.315 2c2.43 0 2.784.013 3.808.06 1.064.049 1.791.218 2.427.465a4.902 4.902 0 011.772 1.153 4.902 4.902 0 011.153 1.772c.247.636.416 1.363.465 2.427.048 1.067.06 1.407.06 4.123v.08c0 2.643-.012 2.987-.06 4.043-.049 1.064-.218 1.791-.465 2.427a4.902 4.902 0 01-1.153 1.772 4.902 4.902 0 01-1.772 1.153c-.636.247-1.363.416-2.427.465-1.067.048-1.407.06-4.123.06h-.08c-2.643 0-2.987-.012-4.043-.06-1.064-.049-1.791-.218-2.427-.465a4.902 4.902 0 01-1.772-1.153 4.902 4.902 0 01-1.153-1.772c-.247-.636-.416-1.363-.465-2.427-.047-1.024-.06-1.379-.06-3.808v-.63c0-2.43.013-2.784.06-3.808.049-1.064.218-1.791.465-2.427a4.902 4.902 0 011.153-1.772A4.902 4.902 0 015.45 2.525c.636-.247 1.363-.416 2.427-.465C8.901 2.013 9.256 2 11.685 2h.63zm-.081 1.802h-.468c-2.456 0-2.784.011-3.807.058-.975.045-1.504.207-1.857.344-.467.182-.8.398-1.15.748-.35.35-.566.683-.748 1.15-.137.353-.3.882-.344 1.857-.047 1.023-.058 1.351-.058 3.807v.468c0 2.456.011 2.784.058 3.807.045.975.207 1.504.344 1.857.182.466.399.8.748 1.15.35.35.683.566 1.15.748.353.137.882.3 1.857.344 1.054.048 1.37.058 4.041.058h.08c2.597 0 2.917-.01 3.96-.058.976-.045 1.505-.207 1.858-.344.466-.182.8-.398 1.15-.748.35-.35.566-.683.748-1.15.137-.353.3-.882.344-1.857.048-1.055.058-1.37.058-4.041v-.08c0-2.597-.01-2.917-.058-3.96-.045-.976-.207-1.505-.344-1.858a3.097 3.097 0 00-.748-1.15 3.098 3.098 0 00-1.15-.748c-.353-.137-.882-.3-1.857-.344-1.023-.047-1.351-.058-3.807-.058zM12 6.865a5.135 5.135 0 110 10.27 5.135 5.135 0 010-10.27zm0 1.802a3.333 3.333 0 100 6.666 3.333 3.333 0 000-6.666zm5.338-3.205a1.2 1.2 0 110 2.4 1.2 1.2 0 010-2.4z6:["$","footer",null,{"className":"bg-gray-900 dark:bg-gray-900","children":["$","div",null,{"className":"mx-auto max-w-7xl px-6 py-4.5 md:flex md:items-center md:justify-between lg:px-8","children":[["$","div",null,{"className":"flex justify-center space-x-6 md:order-2","children":[["$","a","Facebook",{"href":"https://facebook.com/gleeztech","target":"_blank","rel":"noopener noreferrer","className":"text-gray-400 hover:text-gray-500","children":[["$","span",null,{"className":"sr-only","children":"Facebook"}],["$","svg",null,{"className":"h-6 w-6","fill":"currentColor","viewBox":"0 0 24 24","aria-hidden":"true","children":["$","path",null,{"fillRule":"evenodd","d":"M22 12c0-5.523-4.477-10-10-10S2 6.477 2 12c0 4.991 3.657 9.128 8.438 9.878v-6.987h-2.54V12h2.54V9.797c0-2.506 1.492-3.89 3.777-3.89 1.094 0 2.238.195 2.238.195v2.46h-1.26c-1.243 0-1.63.771-1.63 1.562V12h2.773l-.443 2.89h-2.33v6.988C18.343 21.128 22 16.991 22 12z","clipRule":"evenodd"}]}]]}],["$","a","Instagram",{"href":"https://instagram.com/gleeztech","target":"_blank","rel":"noopener noreferrer","className":"text-gray-400 hover:text-gray-500","children":[["$","span",null,{"className":"sr-only","children":"Instagram"}],["$","svg",null,{"className":"h-6 w-6","fill":"currentColor","viewBox":"0 0 24 24","aria-hidden":"true","children":["$","path",null,{"fillRule":"evenodd","d":"$b","clipRule":"evenodd"}]}]]}],["$","a","X",{"href":"https://x.com/gleeztech","target":"_blank","rel":"noopener noreferrer","className":"text-gray-400 hover:text-gray-500","children":[["$","span",null,{"className":"sr-only","children":"X"}],["$","svg",null,{"className":"h-6 w-6","fill":"currentColor","viewBox":"0 0 24 24","aria-hidden":"true","children":["$","path",null,{"d":"M13.6823 10.6218L20.2391 3H18.6854L12.9921 9.61788L8.44486 3H3.2002L10.0765 13.0074L3.2002 21H4.75404L10.7663 14.0113L15.5685 21H20.8131L13.6819 10.6218H13.6823ZM11.5541 13.0956L10.8574 12.0991L5.31391 4.16971H7.70053L12.1742 10.5689L12.8709 11.5655L18.6861 19.8835H16.2995L11.5541 13.096V13.0956Z"}]}]]}],"$Lc"]}],"$Ld"]}]}]
9:null
a:["$","$4",null,{"fallback":["$","div",null,{"className":"comments-skeleton","children":[["$","div",null,{"className":"h-6 bg-gray-200 rounded w-32 mb-4 animate-pulse"}],["$","div",null,{"className":"space-y-3","children":[["$","div","1",{"className":"h-16 bg-gray-100 rounded animate-pulse"}],["$","div","2",{"className":"h-16 bg-gray-100 rounded animate-pulse"}],["$","div","3",{"className":"h-16 bg-gray-100 rounded animate-pulse"}]]}]]}],"children":"$Le"}]
c:["$","a","GitHub",{"href":"https://github.com/gleez","target":"_blank","rel":"noopener noreferrer","className":"text-gray-400 hover:text-gray-500","children":[["$","span",null,{"className":"sr-only","children":"GitHub"}],["$","svg",null,{"className":"h-6 w-6","fill":"currentColor","viewBox":"0 0 24 24","aria-hidden":"true","children":["$","path",null,{"fillRule":"evenodd","d":"M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z","clipRule":"evenodd"}]}]]}]
d:["$","div",null,{"className":"mt-8 md:order-1 md:mt-0","children":["$","p",null,{"className":"text-sm leading-5 text-gray-400","children":["© 2005 - ",2026," Gleez Inc. All rights reserved."]}]}]
e:["$","div",null,{"className":"mt-8","children":[["$","h3",null,{"className":"text-xl font-semibold mb-4","children":"Voices"}],["$","p",null,{"className":"text-gray-500","children":"Your thoughts on \"Further thoughts on hindering screen-scraping\" would be welcome here!"}]]}]