{"id":11120,"date":"2025-03-01T08:25:00","date_gmt":"2025-03-01T08:25:00","guid":{"rendered":"https:\/\/dariusnarmontas.com\/?p=11120"},"modified":"2026-03-02T19:30:13","modified_gmt":"2026-03-02T19:30:13","slug":"new-urban-development-analysis-in-oslo-using-web-scraping-python-selenium-wip","status":"publish","type":"post","link":"https:\/\/dariusnarmontas.com\/index.php\/2025\/03\/01\/new-urban-development-analysis-in-oslo-using-web-scraping-python-selenium-wip\/","title":{"rendered":"New Urban Development Analysis in Oslo using Web Scraping (Python, Selenium) WIP"},"content":{"rendered":"<style>\n       .errordiv { padding:10px; margin:10px; border: 1px solid #555555;color: #000000;background-color: #f8f8f8; width:500px; }#advanced_iframe {visibility:visible;opacity:1;vertical-align:top;}.ai-info-bottom-iframe { position: fixed; z-index: 10000; bottom:0; left: 0; margin: 0px; text-align: center; width: 100%; background-color: #ff9999; padding-left: 5px;padding-bottom: 5px; border-top: 1px solid #aaa } a.ai-bold {font-weight: bold;}#ai-layer-div-advanced_iframe p {height:100%;margin:0;padding:0}<\/style><script type=\"text\/javascript\">var ai_iframe_width_advanced_iframe = 0;var ai_iframe_height_advanced_iframe = 0;function aiReceiveMessageadvanced_iframe(event) {  aiProcessMessage(event,\"advanced_iframe\", \"true\");}if (window.addEventListener) {  window.addEventListener(\"message\", aiReceiveMessageadvanced_iframe);} else if (el.attachEvent)  {  el.attachEvent(\"message\", aiReceiveMessageadvanced_iframe);}var aiOnloadScrollTop=\"true\";var aiShowDebug=false;\n\t\tif (typeof aiReadyCallbacks === 'undefined') {\n\t\t\tvar aiReadyCallbacks = [];\n\t\t} else if (!(aiReadyCallbacks instanceof Array)) {\n\t\t\tvar aiReadyCallbacks = [];\n\t\t}    function aiShowIframeId(id_iframe) { jQuery(\"#\"+id_iframe).css(\"visibility\", \"visible\");    }    function aiResizeIframeHeight(height) { aiResizeIframeHeight(height,advanced_iframe); }    function aiResizeIframeHeightId(height,width,id) {aiResizeIframeHeightById(id,height);}<\/script><iframe id=\"advanced_iframe\"  name=\"advanced_iframe\"  src=\"https:\/\/darmnar.github.io\/finnmap\/\"  width=\"100%\"  height=\"600\"  frameborder=\"0\"  border=\"0\"  allowtransparency=\"true\"  loading=\"lazy\"  style=\";width:100%;height:600px;\" ><\/iframe><script type=\"text\/javascript\">var ifrm_advanced_iframe = document.getElementById(\"advanced_iframe\");var hiddenTabsDoneadvanced_iframe = false;\nfunction resizeCallbackadvanced_iframe() {}<\/script><script type=\"text\/javascript\"><\/script><p style=\"display:block !important; visibility:visible !important;margin: -18px 14px 0 0;padding-left: 3px;padding-top:3px;background: white; overflow: hidden; position: relative; line-height:15px;width: fit-content;\"><small style=\"display:block !important;visibility:visible !important\">powered by Advanced iFrame<\/small><\/p>\n\n\n\n<p class=\"has-text-align-center\">Gathered data displayed on a Google map using geocoded locations (json file) <\/p>\n\n\n<style>\n       .errordiv { padding:10px; margin:10px; border: 1px solid #555555;color: #000000;background-color: #f8f8f8; width:500px; }#advanced_iframe_2 {visibility:visible;opacity:1;vertical-align:top;}.ai-info-bottom-iframe { position: fixed; z-index: 10000; bottom:0; left: 0; margin: 0px; text-align: center; width: 100%; background-color: #ff9999; padding-left: 5px;padding-bottom: 5px; border-top: 1px solid #aaa } a.ai-bold {font-weight: bold;}#ai-layer-div-advanced_iframe_2 p {height:100%;margin:0;padding:0}<\/style><script type=\"text\/javascript\">var ai_iframe_width_advanced_iframe_2 = 0;var ai_iframe_height_advanced_iframe_2 = 0;function aiReceiveMessageadvanced_iframe_2(event) {  aiProcessMessage(event,\"advanced_iframe_2\", \"true\");}if (window.addEventListener) {  window.addEventListener(\"message\", aiReceiveMessageadvanced_iframe_2);} else if (el.attachEvent)  {  el.attachEvent(\"message\", aiReceiveMessageadvanced_iframe_2);}var aiOnloadScrollTop=\"true\";var aiShowDebug=false;\n\t\tif (typeof aiReadyCallbacks === 'undefined') {\n\t\t\tvar aiReadyCallbacks = [];\n\t\t} else if (!(aiReadyCallbacks instanceof Array)) {\n\t\t\tvar aiReadyCallbacks = [];\n\t\t}    function aiShowIframeId(id_iframe) { jQuery(\"#\"+id_iframe).css(\"visibility\", \"visible\");    }    function aiResizeIframeHeight(height) { aiResizeIframeHeight(height,advanced_iframe_2); }    function aiResizeIframeHeightId(height,width,id) {aiResizeIframeHeightById(id,height);}<\/script><iframe id=\"advanced_iframe_2\"  name=\"advanced_iframe_2\"  src=\"https:\/\/darmnar.github.io\/keplerapartments\/\"  width=\"100%\"  height=\"600\"  frameborder=\"0\"  border=\"0\"  allowtransparency=\"true\"  loading=\"lazy\"  style=\";width:100%;height:600px;\" ><\/iframe><script type=\"text\/javascript\">var ifrm_advanced_iframe_2 = document.getElementById(\"advanced_iframe_2\");var hiddenTabsDoneadvanced_iframe_2 = false;\nfunction resizeCallbackadvanced_iframe_2() {}<\/script><script type=\"text\/javascript\"><\/script><p style=\"display:block !important; visibility:visible !important;margin: -18px 14px 0 0;padding-left: 3px;padding-top:3px;background: white; overflow: hidden; position: relative; line-height:15px;width: fit-content;\"><small style=\"display:block !important;visibility:visible !important\">powered by Advanced iFrame<\/small><\/p>\n\n\n\n<div style=\"height:63px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<figure class=\"wp-block-image size-large\"><img fetchpriority=\"high\" decoding=\"async\" width=\"1030\" height=\"494\" src=\"https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/chrome_7qlft2Neap-1030x494.jpg\" alt=\"\" class=\"wp-image-11142\" srcset=\"https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/chrome_7qlft2Neap-1030x494.jpg 1030w, https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/chrome_7qlft2Neap-300x144.jpg 300w, https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/chrome_7qlft2Neap-768x369.jpg 768w, https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/chrome_7qlft2Neap-1536x737.jpg 1536w, https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/chrome_7qlft2Neap.jpg 2000w\" sizes=\"(max-width: 1030px) 100vw, 1030px\" \/><\/figure>\n\n\n\n<p class=\"has-text-align-center\">Corresponding finn.no webpage<\/p>\n\n\n\n<div style=\"height:63px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p>Online marketplace and ad websites contain a lot of valuable information, specifically about new architectural developments. Finn.no is the biggest such website in Norway and it is common if not standard to post real estate ads and information there. <\/p>\n\n\n\n<p>Finn.no offers an API service, but as I am not a business I decided to go around it and make my own web scraper using Python and Selenium.<\/p>\n\n\n\n<p>Looking at how finn.no web links are formatted, we can see that all the filters and parameters are in the link itself, which is of great help  &#8211; https:\/\/www.finn.no\/map\/realestate\/newbuildings?<strong>filters=&amp;lat=59.85704&amp;lon=10.86167&amp;results=true&amp;zoom=9<\/strong><\/p>\n\n\n\n<p>As this changes the data on the website we can easily change and adapt our scraper &#8211; https:\/\/www.finn.no\/map\/realestate\/<strong>businessrent<\/strong>?filters=&amp;lat=59.75312&amp;lon=10.86585&amp;results=true&amp;zoom=10 (now we have results for office rent)<\/p>\n\n\n\n<div style=\"height:43px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p><strong>Example from the dataset:<\/strong><\/p>\n\n\n\n<figure class=\"wp-block-table\"><table><tbody><tr><td><strong>Name<\/strong><\/td><td><strong>Description<\/strong><\/td><td><strong>Price<\/strong><\/td><td><strong>URL<\/strong><\/td><td><strong>Address<\/strong><\/td><\/tr><tr><td><br><br>Vestbyveien<\/td><td><br><br>S\u00d8RUMSAND: Ny enebolig med hybel under oppf\u00f8ring &#8211; 6 sov &#8211; 3 bad &#8211; God standard!<\/td><td><br><br>NA<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/planned\/ad.html?finnkode=399411444<\/td><td><br><br>Vestbyveien 14, 1920 S\u00f8rumsand<\/td><\/tr><tr><td><br><br>Spro Havn &#8211; Seilet A &#8211; C<\/td><td><br><br>SPRO HAVN \/\u200b NESODDEN| Flott, gj.g\u00e5ende 4 roms med utsikt | Solrik, vestvendt balkong | Heis | Mulig garasje-\/\u200bb\u00e5tplass<\/td><td><br><br>6 750 000 kr<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/projectsingle\/ad.html?finnkode=404295059<sup data-fn=\"78a173dc-8b67-40fc-aefd-37ac6978eb49\" class=\"fn\"><a href=\"#78a173dc-8b67-40fc-aefd-37ac6978eb49\" id=\"78a173dc-8b67-40fc-aefd-37ac6978eb49-link\">1<\/a><\/sup><\/td><td><br><br>Julius Rosens vei 35-39, 1454 Fagerstrand<\/td><\/tr><tr><td><br><br>Br\u00e5ten<\/td><td><br><br>Fagerstrand\/\u200bNesodden &#8211; Nytt salgstrinn &#8211; Leiligheter 80-110m2 &#8211; Solrike uteplasser &#8211; 2 solgt<\/td><td><br><br>4 895 000 &#8211; 6 195 000 kr<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=343701712<\/td><td><br><br><br><br>Sagstubben, 1454 Fagerstrand<\/td><\/tr><\/tbody><\/table><figcaption class=\"wp-element-caption\">Example data gathered in first pass<\/figcaption><\/figure>\n\n\n\n<div style=\"height:65px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p>Usually the prices are displayed either as a range or not available, but the full price table is available on the ad  page. We can crawl through the pages again and gather the data:<\/p>\n\n\n\n<div style=\"height:21px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<figure class=\"wp-block-table\"><table><tbody><tr><td><strong>Navn<\/strong><\/td><td><strong>Enhet<\/strong><\/td><td><strong>Etasje<\/strong><\/td><td><strong>BRA-i<\/strong><\/td><td><strong>Soverom<\/strong><\/td><td><strong>Totalpris<\/strong><\/td><td><strong>URL Index<\/strong><\/td><td><strong>Source URL<\/strong><\/td><\/tr><tr><td><br><br>Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230;<\/td><td><br><br>1025<\/td><td>1<\/td><td><br><br>98 m\u00b2<\/td><td>2<\/td><td><br><br>9 954 533 kr<\/td><td><br><br>2<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402<\/td><\/tr><tr><td><br><br>Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230;<\/td><td><br><br>1031<\/td><td>1<\/td><td><br><br>91 m\u00b2<\/td><td>3<\/td><td><br><br>9 001 313 kr<\/td><td><br><br>2<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402<\/td><\/tr><tr><td><br><br>Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230;<\/td><td><br><br>2031<\/td><td>2<\/td><td><br><br>98 m\u00b2<\/td><td>3<\/td><td><br><br>9 404 533 kr<\/td><td><br><br>2<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402<\/td><\/tr><tr><td><br><br>Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230;<\/td><td><br><br>3003<\/td><td>3<\/td><td><br><br>59 m\u00b2<\/td><td>2<\/td><td><br><br>5 836 363 kr<\/td><td><br><br>2<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402<\/td><\/tr><tr><td><br><br>Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230;<\/td><td><br><br>3023<\/td><td>3<\/td><td><br><br>59 m\u00b2<\/td><td>1<\/td><td><br><br>5 636 593 kr<\/td><td><br><br>2<\/td><td><br><br>https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402<\/td><\/tr><\/tbody><\/table><figcaption class=\"wp-element-caption\">Example data table for one project (not complete)<\/figcaption><\/figure>\n\n\n\n<div style=\"height:57px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p>After merging all the data tables together we have <strong>1435<\/strong> entries of apartment data (including geocoded coordinates for google maps). This now allows us to perform data analysis on the gathered dataset (quick examples):<\/p>\n\n\n\n<div style=\"height:38px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" width=\"1006\" height=\"556\" src=\"https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/01_finn_scrape_data.jpg\" alt=\"\" class=\"wp-image-11126\" srcset=\"https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/01_finn_scrape_data.jpg 1006w, https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/01_finn_scrape_data-300x166.jpg 300w, https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/01_finn_scrape_data-768x424.jpg 768w\" sizes=\"(max-width: 1006px) 100vw, 1006px\" \/><\/figure>\n\n\n\n<div style=\"height:100px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<figure class=\"wp-block-image size-full\"><img decoding=\"async\" width=\"645\" height=\"532\" src=\"https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/02_finn_scrape_data.jpg\" alt=\"\" class=\"wp-image-11127\" srcset=\"https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/02_finn_scrape_data.jpg 645w, https:\/\/dariusnarmontas.com\/wp-content\/uploads\/2025\/05\/02_finn_scrape_data-300x247.jpg 300w\" sizes=\"(max-width: 645px) 100vw, 645px\" \/><\/figure>\n\n\n\n<div style=\"height:86px\" aria-hidden=\"true\" class=\"wp-block-spacer\"><\/div>\n\n\n\n<p>The next step would be to bring all this data into Rhino 3D with grasshopper and \/ or try out Qgis to better visualise the data, create heatmaps and so on.<\/p>\n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"<p>Gathered data displayed on a Google map using geocoded locations (json file) Corresponding finn.no webpage Online marketplace and ad websites contain a lot of valuable information, specifically about new architectural developments. Finn.no is the biggest such website in Norway and it is common if not standard to post real estate ads and information there. Finn.no offers an API service, but as I am not a business I decided to go around it and make my own web scraper using Python and Selenium. Looking at how finn.no web links are formatted, we can see that all the filters and parameters are in the link itself, which is of great help &#8211; https:\/\/www.finn.no\/map\/realestate\/newbuildings?filters=&amp;lat=59.85704&amp;lon=10.86167&amp;results=true&amp;zoom=9 As this changes the data on the website we can easily change and adapt our scraper &#8211; https:\/\/www.finn.no\/map\/realestate\/businessrent?filters=&amp;lat=59.75312&amp;lon=10.86585&amp;results=true&amp;zoom=10 (now we have results for office rent) Example from the dataset: Name Description Price URL Address Vestbyveien S\u00d8RUMSAND: Ny enebolig med hybel under oppf\u00f8ring &#8211; 6 sov &#8211; 3 bad &#8211; God standard! NA https:\/\/www.finn.no\/realestate\/planned\/ad.html?finnkode=399411444 Vestbyveien 14, 1920 S\u00f8rumsand Spro Havn &#8211; Seilet A &#8211; C SPRO HAVN \/\u200b NESODDEN| Flott, gj.g\u00e5ende 4 roms med utsikt | Solrik, vestvendt balkong | Heis | Mulig garasje-\/\u200bb\u00e5tplass 6 750 000 kr https:\/\/www.finn.no\/realestate\/projectsingle\/ad.html?finnkode=404295059 Julius Rosens vei 35-39, 1454 Fagerstrand Br\u00e5ten Fagerstrand\/\u200bNesodden &#8211; Nytt salgstrinn &#8211; Leiligheter 80-110m2 &#8211; Solrike uteplasser &#8211; 2 solgt 4 895 000 &#8211; 6 195 000 kr https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=343701712 Sagstubben, 1454 Fagerstrand Example data gathered in first pass Usually the prices are displayed either as a range or not available, but the full price table is available on the ad page. We can crawl through the pages again and gather the data: Navn Enhet Etasje BRA-i Soverom Totalpris URL Index Source URL Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230; 1025 1 98 m\u00b2 2 9 954 533 kr 2 https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402 Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230; 1031 1 91 m\u00b2 3 9 001 313 kr 2 https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402 Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230; 2031 2 98 m\u00b2 3 9 404 533 kr 2 https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402 Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230; 3003 3 59 m\u00b2 2 5 836 363 kr 2 https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402 Nytt salgstrinn i Ulvenplassen Humleblom borettslag, salgstrinn 2. Salgsstart&#8230; 3023 3 59 m\u00b2 1 5 636 593 kr 2 https:\/\/www.finn.no\/realestate\/project\/ad.html?finnkode=404292402 Example data table for one project (not complete) After merging all the data tables together we have 1435 entries of apartment data (including geocoded coordinates for google maps). This now allows us to perform data analysis on the gathered dataset (quick examples): The next step would be to bring all this data into Rhino 3D with grasshopper and \/ or try out Qgis to better visualise the data, create heatmaps and so on.<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_monsterinsights_skip_tracking":false,"_monsterinsights_sitenote_active":false,"_monsterinsights_sitenote_note":"","_monsterinsights_sitenote_category":0,"footnotes":"[{\"id\":\"78a173dc-8b67-40fc-aefd-37ac6978eb49\",\"content\":\"\"}]"},"categories":[1],"tags":[94,95,97,98],"class_list":["post-11120","post","type-post","status-publish","format-standard","hentry","category-blog","tag-computation","tag-data-analysis","tag-python","tag-web-scraping"],"aioseo_notices":[],"_links":{"self":[{"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/posts\/11120","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/comments?post=11120"}],"version-history":[{"count":14,"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/posts\/11120\/revisions"}],"predecessor-version":[{"id":11165,"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/posts\/11120\/revisions\/11165"}],"wp:attachment":[{"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/media?parent=11120"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/categories?post=11120"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/dariusnarmontas.com\/index.php\/wp-json\/wp\/v2\/tags?post=11120"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}