VBA webscraping - can't get innerText from class

109 Views Asked by At

i need to get copper price from website, but I get {{price}} instead of value. What I'm doing incorrectly? Thanks for help!

Option Explicit

Sub Macro1()
    
    Const URL As String _
        = "https://www.lme.com/"
    
    Const ClassName As String _
        = "metal-block__price"
    
    Dim WhrResponseText As String
    WhrResponseText = GetWhrResponseText(URL)
    If Len(WhrResponseText) = 0 Then
        MsgBox "Could not get a response.", vbExclamation
        Exit Sub
    End If

    Dim Elements As Object
    With CreateObject("htmlfile")
        .body.innerHTML = WhrResponseText
        Set Elements = .getElementsByClassName(ClassName)
    End With
    
    Dim Result As Variant
    With Elements
        If .Length > 0 Then
            Result = .Item(0).innerText
            MsgBox Result
        Else
            MsgBox "Nothing found."
        End If
    End With
        
    Dim i As Long
    
    Dim Element As Object
    For Each Element In Elements
        Debug.Print i, Element.innerText
        i = i + 1
    Next Element

End Sub


Function GetWhrResponseText( _
    ByVal URL As String) _
As String
    Const ProcName As String = "GetWhrResponseText"
    On Error GoTo ClearError

    With CreateObject("WinHttp.WinHttpRequest.5.1")
        .Open "GET", URL, False
        .send
        GetWhrResponseText = StrConv(.responseBody, vbUnicode)
    End With

ProcExit:
    Exit Function
ClearError:
    Debug.Print "'" & ProcName & "' Run-time error '" _
        & Err.Number & "':" & vbLf & "    " & Err.Description
    Resume ProcExit
End Function

Expected clarification why it works on other websites but not on this one.

1

There are 1 best solutions below

0
SIM On

This is one of the ways you can scrape prices from inner pages after you have traversed the links attached to the price available on the landing page.

Option Explicit
Sub CollectPrice()
    Const startUrl$ = "https://www.lme.com/"
    Dim oHttp As Object, innerLink$, price$
    Dim Html As HTMLDocument, sBase$, I&
    
    sBase = "https://www.lme.com"
    
    Set oHttp = CreateObject("MSXML2.XMLHTTP")
    Set Html = New HTMLDocument
    
    With oHttp
        .Open "GET", startUrl, True
        .setRequestHeader "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"
        .send
        While .readyState < 4: DoEvents: Wend
        Html.body.innerHTML = .responseText
        With Html.querySelectorAll("a.metal-block")
            For I = 0 To .Length - 1
                innerLink = sBase & .Item(I).getAttribute("href")
                With oHttp
                    .Open "GET", innerLink, True
                    .setRequestHeader "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"
                    .send
                    While .readyState < 4: DoEvents: Wend
                    Html.body.innerHTML = .responseText
                    price = Html.querySelector("span.hero-metal-data__number").innerText
                    Debug.Print price
                End With
            Next I
        End With
    End With
End Sub

Make sure to add this reference, Microsoft HTML Object Library to the library in order for the script to work.