modulos_internal/internal/thelist.php

<?php
/* 
 * thelist web scraper for Land Title Details
 * Benjamin Harris, last updated 20/12/2021
 * https://sms.bisonconstructions.com.au
*/
date_default_timezone_set("Australia/Hobart");

error_reporting(E_ALL);
ini_set('display_errors', '0');
ini_set('log_errors', '1');
//error_reporting(E_ERROR | E_PARSE);

// Function to perform a POST request for login
function login($username, $password) {
    $loginUrl = 'https://security.thelist.tas.gov.au/cas/login'; // Updated login URL
    $loginData = array(
        'username' => $username,
        'password' => $password
    	//'lt' => 'LT-2965-PYQs5SPwZuphpeuhmfQOwvHjfY3KQ6', // these are hidden inputs in the login form.
        //'execution' => 'e2s1', 
        //'_eventId' => 'submit' 
    );

    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $loginUrl);
    curl_setopt($ch, CURLOPT_POST, true);
    curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($loginData));
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

    // Execute the login request
    $loginResponse = curl_exec($ch);

    // Check for errors and look for the success message
    if (curl_errno($ch)) {
        echo 'Error: ' . curl_error($ch);
    } elseif (strpos($loginResponse, '<div id="msg" class="success">') !== false) {
        echo 'Logged in successfully.';
    } else {
        echo 'Login failed.';
    }

    curl_close($ch);
}

// Function to scrape data from the search page
function scrapeData($username, $password, $streetNumber, $streetName, $streetType, $locality) {
    // Call the login function to authenticate
    login($username, $password);

    // URL for the search with parameters
    $searchUrl = "https://www.thelist.tas.gov.au/app/content/property/property-search?streetNumber=$streetNumber&propertySearchCriteria.streetName=$streetName&propertySearchCriteria.streetType=$streetType&propertySearchCriteria.locality=$locality";

    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $searchUrl);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

    // Execute the search request
    $searchResponse = curl_exec($ch);

    // Check for errors and parse the HTML response to extract the required data
    if (curl_errno($ch)) {
        echo 'Error: ' . curl_error($ch);
    } else {
        // Parse the HTML response and extract the required data
        $doc = new DOMDocument();
        @$doc->loadHTML($searchResponse); // Suppress errors for invalid HTML

        // Use XPath to extract specific elements
        $xpath = new DOMXPath($doc);
        $propertyListTitle = $xpath->query('//td[@aria-describedby="propertyList_title"]')->item(0)->textContent;
        $propertyListPropertyId = $xpath->query('//td[@aria-describedby="propertyList_propertyId"]')->item(0)->textContent;
        $propertyList1TName = $xpath->query('//td[@aria-describedby="propertyList_1_t_name"]')->item(0)->textContent;

        echo "propertyList_title: $propertyListTitle\n";
        echo "propertyList_propertyId: $propertyListPropertyId\n";
        echo "propertyList_1_t_name: $propertyList1TName\n";
    }

    curl_close($ch);
}

// Usage
$username = 'david@bisonent.com.au';
$password = 'takecare';
$streetNumber = '29';
$streetName = 'ALFRED';
$streetType = 'ST';
$locality = 'SCOTTSDALE';

scrapeData($username, $password, $streetNumber, $streetName, $streetType, $locality);

?>