convert InternetFileRead() to unicode Topic is solved

Get help with using AutoHotkey (v1.1 and older) and its commands and hotkeys
guest3456
Posts: 3454
Joined: 09 Oct 2013, 10:31

convert InternetFileRead() to unicode

06 Dec 2019, 20:51

this works on ansi. can someone help me convert it so it works on unicode too? i tried removing the "A"s from the 3 functions, but then i just get chinese chars. the return value of the func is "bytes read" and its one byte larger on unicode, if that matters

Code: Select all

msgbox, % InternetFileRead(ver, "https://raw.githubusercontent.com/Lexikos/AutoHotkey_L-Docs/master/index.html", 1024, 1024, "")
msgbox, ver=%ver%


;// https://autohotkey.com/board/topic/42096-function-internetfileread-aka-urldownloadtovar/page-1
InternetFileRead( ByRef V, URL="", RB=0, bSz=1024, DLP="DLP", F=0x84000000 ) {
 Static LIB="WININET\", CL="00000000000000", N=""
 QRL := 16
 If ! DllCall( "GetModuleHandle", Str,"wininet.dll" )
      DllCall( "LoadLibrary", Str,"wininet.dll" )
 If ! hIO:=DllCall( LIB "InternetOpenA", Str,N, UInt,4, Str,N, Str,N, UInt,0 )
   Return -1
 If ! (( hIU:=DllCall( LIB "InternetOpenUrlA", UInt,hIO, Str,URL, Str,N, Int,0, UInt,F
                                                            , UInt,0 ) ) || ErrorLevel )
   Return 0 - ( !DllCall( LIB "InternetCloseHandle", UInt,hIO ) ) - 2
 If ! ( RB  )
 If ( SubStr(URL,1,4) = "ftp:" )
    CL := DllCall( LIB "FtpGetFileSize", UInt,hIU, UIntP,0 )
 Else If ! DllCall( LIB "HttpQueryInfoA", UInt,hIU, Int,5, Str,CL, UIntP,QRL, UInt,0 )
   Return 0 - ( !DllCall( LIB "InternetCloseHandle", UInt,hIU ) )
            - ( !DllCall( LIB "InternetCloseHandle", UInt,hIO ) ) - 4
 VarSetCapacity( V,64 ), VarSetCapacity( V,0 )
 SplitPath, URL, FN,,,, DN
 FN:=(FN ? FN : DN), CL:=(RB ? RB : CL), VarSetCapacity( V,CL,32 ), P:=&V,
 B:=(bSz>CL ? CL : bSz), TtlB:=0, LP := RB ? "Unknown" : CL,  %DLP%( True,CL,FN )
 Loop {
       If ( DllCall( LIB "InternetReadFile", UInt,hIU, UInt,P, UInt,B, UIntP,R ) && !R )
       Break
       P:=(P+R), TtlB:=(TtlB+R), RemB:=(CL-TtlB), B:=(RemB<B ? RemB : B), %DLP%( TtlB,LP )
       Sleep -1
 } TtlB<>CL ? VarSetCapacity( T,TtlB ) DllCall( "RtlMoveMemory", Str,T, Str,V, UInt,TtlB )
  . VarSetCapacity( V,0 ) . VarSetCapacity( V,TtlB,32 ) . DllCall( "RtlMoveMemory", Str,V
  , Str,T, UInt,TtlB ) . %DLP%( TtlB, TtlB ) : N
 If ( !DllCall( LIB "InternetCloseHandle", UInt,hIU ) )
  + ( !DllCall( LIB "InternetCloseHandle", UInt,hIO ) )
   Return -6
Return, VarSetCapacity(V)+((ErrorLevel:=(RB>0 && TtlB<RB)||(RB=0 && TtlB=CL) ? 0 : 1)<<64)
}
i'm trying to avoid the COM solutions and just want to see if i can get this to work

teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode  Topic is solved

07 Dec 2019, 06:14

I use this code, it's pretty reliable:

Code: Select all

url := "https://raw.githubusercontent.com/Lexikos/AutoHotkey_L-Docs/master/index.html"
if length := DataFromUrl(url, data) {
   html := StrGet(&data, length, "utf-8")
   MsgBox, % html
}

DataFromUrl(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
        , FILE_BEGIN := 0, INVALID_SET_FILE_POINTER := -1, NO_ERROR := 0
        , HEAP_ZERO_MEMORY := 0x8
        
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", "Str", "Wininet.dll", "Ptr")
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", "Str", userAgent, "UInt", INTERNET_OPEN_TYPE_DIRECT, "Ptr", 0, "Ptr", 0, "UInt", 0, "Ptr") {
         error := "InternetOpen failed. Last Error: " . A_LastError
         break
      }
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         while !hUrl := DllCall("Wininet\InternetOpenUrl", "Ptr", hInternet, "Str", url, "Ptr", 0, "UInt", 0, "UInt", OpenUrlFlags, "Ptr", 0, "Ptr") {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl failed. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", "Ptr", hUrl, "UInt", QueryInfoFlags, "UIntP", fullSize, "UIntP", l := 4, "UIntP", idx := 0)
            hHeap := DllCall("GetProcessHeap", "Ptr")
            pHeap := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize := fullSize ? fullSize : 0x100000, "Ptr")
            prevSize := bytesRead := 0
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", "Ptr", hUrl, "UInt", bytesRead & 0xFFFFFFFF
                                                           , "IntP", h := (bytesRead >> 32), "UInt", FILE_BEGIN, "UInt", 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer failed. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", "Ptr", hUrl, "UIntP", size, "UInt", 0, "Ptr", 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize *= 2, "Ptr")
               DllCall("RtlMoveMemory", "Ptr", pHeapNew, "Ptr", pHeap, "Ptr", bytesRead)
               DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", "Ptr", hUrl, "Ptr", pHeap + bytesRead, "UInt", size, "UIntP", read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", "Ptr", hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", "Ptr", hInternet) )
   DllCall("FreeLibrary", "Ptr", hLib)
   if error {
      DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", "Ptr", &data, "Ptr", pHeap, "Ptr", bytesRead)
   }
   DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
   Return bytesRead
}
guest3456
Posts: 3454
Joined: 09 Oct 2013, 10:31

Re: convert InternetFileRead() to unicode

07 Dec 2019, 09:50

teadrinker wrote:
07 Dec 2019, 06:14
I use this code, it's pretty reliable:

Code: Select all

url := "https://raw.githubusercontent.com/Lexikos/AutoHotkey_L-Docs/master/index.html"
if length := DataFromUrl(url, data) {
   html := StrGet(&data, length, "utf-8")
   MsgBox, % html
}

DataFromUrl(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
   ;...
  
lol thanks, using StrGet ALSO works with the other func:

Code: Select all

msgbox, % length := InternetFileRead(ver, "https://raw.githubusercontent.com/Lexikos/AutoHotkey_L-Docs/master/index.html", 1024, 1024, "")
msgbox, % StrGet(&ver, length, "utf-8")

finally do you prefer using this over the various COM methods? i'm wondering if you have any experience with the COM methods not working as well as this or not?

teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

07 Dec 2019, 10:39

For me COM methods work well, but some people complain that COM methods don't work on their machines while winapi method works. Maybe that because of their registry or IE settings.
Last edited by teadrinker on 07 Dec 2019, 10:42, edited 1 time in total.
guest3456
Posts: 3454
Joined: 09 Oct 2013, 10:31

Re: convert InternetFileRead() to unicode

07 Dec 2019, 10:41

teadrinker wrote:
07 Dec 2019, 10:39
For me COM methods work well, but some people complain that COM methods doesn't work on their machines while winapi method works. Maybe that because of their registry or IE settings.
ok gotcha. i will say that the InternetFileRead() that i've used above DID have some problems with HTTPS links on some win7s. i had to instruct those users to turn on TLS1.2 within IE settings (Internet Options Control Panel) Advanced tab. maybe your func is similar.

tmplinshi
Posts: 1604
Joined: 01 Oct 2013, 14:57

Re: convert InternetFileRead() to unicode

07 Dec 2019, 11:28

Hello, I added gzip decoding support :wave:
(Changes I made: Added InternetSetOption, and modified InternetOpenUrl)

Code: Select all

url := "https://httpbin.org/brotli"
url := "https://httpbin.org/gzip"
if length := DataFromUrl_tmp(url, data) {
   html := StrGet(&data, length, "utf-8")
   MsgBox, % html
}

DataFromUrl_tmp(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1, INTERNET_OPTION_HTTP_DECODING := 65
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", Str, "Wininet.dll", Ptr)
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", Str, userAgent, UInt, INTERNET_OPEN_TYPE_DIRECT, Ptr, 0, Ptr, 0, UInt, 0, Ptr) {
         error := "InternetOpen fails. Last Error: " . A_LastError
         break
      }
      DllCall("Wininet\InternetSetOption", Ptr, hInternet, UInt, INTERNET_OPTION_HTTP_DECODING, PtrP, lpBuffer:=1, UInt, 4)
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         headers := "Accept-Encoding: gzip, deflate, br"
         while !hUrl := DllCall("Wininet\InternetOpenUrl", Ptr, hInternet, Str, url, Str, headers, UInt, StrLen(headers), UInt, OpenUrlFlags, Ptr, 0, Ptr) {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl fails. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", Ptr, hUrl, UInt, QueryInfoFlags, UIntP, fullSize, UIntP, l := 4, UIntP, idx := 0)
            hHeap := DllCall("GetProcessHeap", Ptr)
            pHeap := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize := fullSize ? fullSize : 0x100000, Ptr)
            prevSize := bytesRead := 0, start := A_TickCount
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", Ptr, hUrl, UInt, bytesRead & 0xFFFFFFFF
                                                                      , IntP, h := (bytesRead >> 32), UInt, FILE_BEGIN, UInt, 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer fails. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", Ptr, hUrl, UIntP, size, UInt, 0, Ptr, 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize *= 2, Ptr)
               DllCall("RtlMoveMemory", Ptr, pHeapNew, Ptr, pHeap, Ptr, bytesRead)
               DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", Ptr, hUrl, Ptr, pHeap + bytesRead, UInt, size, UIntP, read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", Ptr, hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", Ptr, hInternet) )
   ( hLib && DllCall("FreeLibrary", Ptr, hLib) )
   if error {
      DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", Ptr, &data, Ptr, pHeap, Ptr, bytesRead)
   }
   DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
   Return bytesRead
}
teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

07 Dec 2019, 12:03

Hi, @tmplinshi

But I can't see the difference:

Code: Select all

url := "https://imgur.com/"
MsgBox, % length := DataFromUrl(url, data)
MsgBox, % length := DataFromUrl_tmp(url, data)

DataFromUrl(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
        , FILE_BEGIN := 0, INVALID_SET_FILE_POINTER := -1, NO_ERROR := 0
        , HEAP_ZERO_MEMORY := 0x8
        
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", "Str", "Wininet.dll", "Ptr")
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", "Str", userAgent, "UInt", INTERNET_OPEN_TYPE_DIRECT, "Ptr", 0, "Ptr", 0, "UInt", 0, "Ptr") {
         error := "InternetOpen failed. Last Error: " . A_LastError
         break
      }
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         while !hUrl := DllCall("Wininet\InternetOpenUrl", "Ptr", hInternet, "Str", url, "Ptr", 0, "UInt", 0, "UInt", OpenUrlFlags, "Ptr", 0, "Ptr") {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl failed. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", "Ptr", hUrl, "UInt", QueryInfoFlags, "UIntP", fullSize, "UIntP", l := 4, "UIntP", idx := 0)
            hHeap := DllCall("GetProcessHeap", "Ptr")
            pHeap := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize := fullSize ? fullSize : 0x100000, "Ptr")
            prevSize := bytesRead := 0
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", "Ptr", hUrl, "UInt", bytesRead & 0xFFFFFFFF
                                                           , "IntP", h := (bytesRead >> 32), "UInt", FILE_BEGIN, "UInt", 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer failed. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", "Ptr", hUrl, "UIntP", size, "UInt", 0, "Ptr", 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize *= 2, "Ptr")
               DllCall("RtlMoveMemory", "Ptr", pHeapNew, "Ptr", pHeap, "Ptr", bytesRead)
               DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", "Ptr", hUrl, "Ptr", pHeap + bytesRead, "UInt", size, "UIntP", read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", "Ptr", hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", "Ptr", hInternet) )
   DllCall("FreeLibrary", "Ptr", hLib)
   if error {
      DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", "Ptr", &data, "Ptr", pHeap, "Ptr", bytesRead)
   }
   DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
   Return bytesRead
}

DataFromUrl_tmp(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1, INTERNET_OPTION_HTTP_DECODING := 65
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", Str, "Wininet.dll", Ptr)
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", Str, userAgent, UInt, INTERNET_OPEN_TYPE_DIRECT, Ptr, 0, Ptr, 0, UInt, 0, Ptr) {
         error := "InternetOpen fails. Last Error: " . A_LastError
         break
      }
      DllCall("Wininet\InternetSetOption", Ptr, hInternet, UInt, INTERNET_OPTION_HTTP_DECODING, PtrP, lpBuffer:=1, UInt, 4)
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         headers := "Accept-Encoding: gzip, deflate, br"
         while !hUrl := DllCall("Wininet\InternetOpenUrl", Ptr, hInternet, Str, url, Str, headers, UInt, StrLen(headers), UInt, OpenUrlFlags, Ptr, 0, Ptr) {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl fails. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", Ptr, hUrl, UInt, QueryInfoFlags, UIntP, fullSize, UIntP, l := 4, UIntP, idx := 0)
            hHeap := DllCall("GetProcessHeap", Ptr)
            pHeap := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize := fullSize ? fullSize : 0x100000, Ptr)
            prevSize := bytesRead := 0, start := A_TickCount
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", Ptr, hUrl, UInt, bytesRead & 0xFFFFFFFF
                                                                      , IntP, h := (bytesRead >> 32), UInt, FILE_BEGIN, UInt, 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer fails. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", Ptr, hUrl, UIntP, size, UInt, 0, Ptr, 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize *= 2, Ptr)
               DllCall("RtlMoveMemory", Ptr, pHeapNew, Ptr, pHeap, Ptr, bytesRead)
               DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", Ptr, hUrl, Ptr, pHeap + bytesRead, UInt, size, UIntP, read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", Ptr, hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", Ptr, hInternet) )
   ( hLib && DllCall("FreeLibrary", Ptr, hLib) )
   if error {
      DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", Ptr, &data, Ptr, pHeap, Ptr, bytesRead)
   }
   DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
   Return bytesRead
}
Or I do something wrong?
teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

07 Dec 2019, 12:23

Ok, it was a bad example. Let's try YouTube:

Code: Select all

url := "https://www.youtube.com/"
MsgBox, % length := DataFromUrl(url, data)
MsgBox, % StrGet(&data, length, "UTF-8")

MsgBox, % length := DataFromUrl_tmp(url, data)
MsgBox, % StrGet(&data, length, "UTF-8")

DataFromUrl(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
        , FILE_BEGIN := 0, INVALID_SET_FILE_POINTER := -1, NO_ERROR := 0
        , HEAP_ZERO_MEMORY := 0x8
        
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", "Str", "Wininet.dll", "Ptr")
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", "Str", userAgent, "UInt", INTERNET_OPEN_TYPE_DIRECT, "Ptr", 0, "Ptr", 0, "UInt", 0, "Ptr") {
         error := "InternetOpen failed. Last Error: " . A_LastError
         break
      }
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         while !hUrl := DllCall("Wininet\InternetOpenUrl", "Ptr", hInternet, "Str", url, "Ptr", 0, "UInt", 0, "UInt", OpenUrlFlags, "Ptr", 0, "Ptr") {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl failed. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", "Ptr", hUrl, "UInt", QueryInfoFlags, "UIntP", fullSize, "UIntP", l := 4, "UIntP", idx := 0)
            hHeap := DllCall("GetProcessHeap", "Ptr")
            pHeap := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize := fullSize ? fullSize : 0x100000, "Ptr")
            prevSize := bytesRead := 0
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", "Ptr", hUrl, "UInt", bytesRead & 0xFFFFFFFF
                                                           , "IntP", h := (bytesRead >> 32), "UInt", FILE_BEGIN, "UInt", 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer failed. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", "Ptr", hUrl, "UIntP", size, "UInt", 0, "Ptr", 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize *= 2, "Ptr")
               DllCall("RtlMoveMemory", "Ptr", pHeapNew, "Ptr", pHeap, "Ptr", bytesRead)
               DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", "Ptr", hUrl, "Ptr", pHeap + bytesRead, "UInt", size, "UIntP", read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", "Ptr", hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", "Ptr", hInternet) )
   DllCall("FreeLibrary", "Ptr", hLib)
   if error {
      DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", "Ptr", &data, "Ptr", pHeap, "Ptr", bytesRead)
   }
   DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
   Return bytesRead
}

DataFromUrl_tmp(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1, INTERNET_OPTION_HTTP_DECODING := 65
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", Str, "Wininet.dll", Ptr)
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", Str, userAgent, UInt, INTERNET_OPEN_TYPE_DIRECT, Ptr, 0, Ptr, 0, UInt, 0, Ptr) {
         error := "InternetOpen fails. Last Error: " . A_LastError
         break
      }
      DllCall("Wininet\InternetSetOption", Ptr, hInternet, UInt, INTERNET_OPTION_HTTP_DECODING, PtrP, lpBuffer:=1, UInt, 4)
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         headers := "Accept-Encoding: gzip, deflate, br"
         while !hUrl := DllCall("Wininet\InternetOpenUrl", Ptr, hInternet, Str, url, Str, headers, UInt, StrLen(headers), UInt, OpenUrlFlags, Ptr, 0, Ptr) {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl fails. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", Ptr, hUrl, UInt, QueryInfoFlags, UIntP, fullSize, UIntP, l := 4, UIntP, idx := 0)
            hHeap := DllCall("GetProcessHeap", Ptr)
            pHeap := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize := fullSize ? fullSize : 0x100000, Ptr)
            prevSize := bytesRead := 0, start := A_TickCount
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", Ptr, hUrl, UInt, bytesRead & 0xFFFFFFFF
                                                                      , IntP, h := (bytesRead >> 32), UInt, FILE_BEGIN, UInt, 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer fails. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", Ptr, hUrl, UIntP, size, UInt, 0, Ptr, 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize *= 2, Ptr)
               DllCall("RtlMoveMemory", Ptr, pHeapNew, Ptr, pHeap, Ptr, bytesRead)
               DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", Ptr, hUrl, Ptr, pHeap + bytesRead, UInt, size, UIntP, read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", Ptr, hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", Ptr, hInternet) )
   ( hLib && DllCall("FreeLibrary", Ptr, hLib) )
   if error {
      DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", Ptr, &data, Ptr, pHeap, Ptr, bytesRead)
   }
   DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
   Return bytesRead
}
There is a difference, but html is not actually decoded.
teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

07 Dec 2019, 12:27

As I understand, to decompress data zlib1.dll is needed.
tmplinshi
Posts: 1604
Joined: 01 Oct 2013, 14:57

Re: convert InternetFileRead() to unicode

07 Dec 2019, 12:41

The length returned by DataFromUrl_tmp is already gzip decoded, so the length will be the same as your original function.
If you comment the line DllCall("Wininet\InternetSetOption"...., you'll see the size will be different, but wait, you can't use "https://imgur.com/" to test, because this url will never return a gzip content whether you've sent a "Accept-Encoding: gzip, deflate" header or not.

Code: Select all

url := "https://www.autohotkey.com/boards/index.php"
MsgBox, % length := DataFromUrl(url, data)
MsgBox, % length := DataFromUrl_tmp(url, data) ; Note: The line `DllCall("Wininet\InternetSetOption"...` has been commented.

DataFromUrl(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
        , FILE_BEGIN := 0, INVALID_SET_FILE_POINTER := -1, NO_ERROR := 0
        , HEAP_ZERO_MEMORY := 0x8
        
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", "Str", "Wininet.dll", "Ptr")
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", "Str", userAgent, "UInt", INTERNET_OPEN_TYPE_DIRECT, "Ptr", 0, "Ptr", 0, "UInt", 0, "Ptr") {
         error := "InternetOpen failed. Last Error: " . A_LastError
         break
      }
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         while !hUrl := DllCall("Wininet\InternetOpenUrl", "Ptr", hInternet, "Str", url, "Ptr", 0, "UInt", 0, "UInt", OpenUrlFlags, "Ptr", 0, "Ptr") {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl failed. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", "Ptr", hUrl, "UInt", QueryInfoFlags, "UIntP", fullSize, "UIntP", l := 4, "UIntP", idx := 0)
            hHeap := DllCall("GetProcessHeap", "Ptr")
            pHeap := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize := fullSize ? fullSize : 0x100000, "Ptr")
            prevSize := bytesRead := 0
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", "Ptr", hUrl, "UInt", bytesRead & 0xFFFFFFFF
                                                           , "IntP", h := (bytesRead >> 32), "UInt", FILE_BEGIN, "UInt", 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer failed. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", "Ptr", hUrl, "UIntP", size, "UInt", 0, "Ptr", 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", "Ptr", hHeap, "UInt", HEAP_ZERO_MEMORY, "Ptr", buffSize *= 2, "Ptr")
               DllCall("RtlMoveMemory", "Ptr", pHeapNew, "Ptr", pHeap, "Ptr", bytesRead)
               DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", "Ptr", hUrl, "Ptr", pHeap + bytesRead, "UInt", size, "UIntP", read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", "Ptr", hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", "Ptr", hInternet) )
   DllCall("FreeLibrary", "Ptr", hLib)
   if error {
      DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", "Ptr", &data, "Ptr", pHeap, "Ptr", bytesRead)
   }
   DllCall("HeapFree", "Ptr", hHeap, "UInt", 0, "Ptr", pHeap)
   Return bytesRead
}

DataFromUrl_tmp(url, ByRef data) {
   static userAgent := "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
        , INTERNET_OPEN_TYPE_DIRECT := 1, INTERNET_OPTION_HTTP_DECODING := 65
        , OpenUrlFlags   := ( INTERNET_FLAG_IGNORE_CERT_CN_INVALID   := 0x00001000 )
                          | ( INTERNET_FLAG_IGNORE_CERT_DATE_INVALID := 0x00002000 )
                          | ( INTERNET_FLAG_RELOAD                   := 0x80000000 )
        , QueryInfoFlags := ( HTTP_QUERY_FLAG_NUMBER                 := 0x20000000 )
                          | ( HTTP_QUERY_CONTENT_LENGTH              := 0x00000005 )
   #MaxMem 4095
   if !hLib := DllCall("LoadLibrary", Str, "Wininet.dll", Ptr)
      throw Exception("Can't load Wininet.dll")

   Loop 1 {
      if !hInternet := DllCall("Wininet\InternetOpen", Str, userAgent, UInt, INTERNET_OPEN_TYPE_DIRECT, Ptr, 0, Ptr, 0, UInt, 0, Ptr) {
         error := "InternetOpen fails. Last Error: " . A_LastError
         break
      }
      ;DllCall("Wininet\InternetSetOption", Ptr, hInternet, UInt, INTERNET_OPTION_HTTP_DECODING, PtrP, lpBuffer:=1, UInt, 4)
      Loop {
         attemptCount := A_Index
         start := A_TickCount
         headers := "Accept-Encoding: gzip, deflate, br"
         while !hUrl := DllCall("Wininet\InternetOpenUrl", Ptr, hInternet, Str, url, Str, headers, UInt, StrLen(headers), UInt, OpenUrlFlags, Ptr, 0, Ptr) {
            if (A_TickCount - start > 30000 && error := "InternetOpenUrl fails. LastError: " . A_LastError . "`nCheck the internet connection!")
               break 2
            Sleep, 1000
         }
         if (attemptCount = 1) {
            DllCall("Wininet\HttpQueryInfo", Ptr, hUrl, UInt, QueryInfoFlags, UIntP, fullSize, UIntP, l := 4, UIntP, idx := 0)
            hHeap := DllCall("GetProcessHeap", Ptr)
            pHeap := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize := fullSize ? fullSize : 0x100000, Ptr)
            prevSize := bytesRead := 0, start := A_TickCount
         }
         else {
            res := DllCall("Wininet\InternetSetFilePointer", Ptr, hUrl, UInt, bytesRead & 0xFFFFFFFF
                                                                      , IntP, h := (bytesRead >> 32), UInt, FILE_BEGIN, UInt, 0)
            if (res = INVALID_SET_FILE_POINTER && A_LastError != NO_ERROR) {
               error := "InternetSetFilePointer fails. Last Error: " . A_LastError
               break 2
            }
         }
         Loop {
            if !DllCall("Wininet\InternetQueryDataAvailable", Ptr, hUrl, UIntP, size, UInt, 0, Ptr, 0)
               break
            if (size = 0 && EOF := true)
               break
            if (bytesRead + size > buffSize) {
               pHeapNew := DllCall("HeapAlloc", Ptr, hHeap, UInt, HEAP_ZERO_MEMORY, Ptr, buffSize *= 2, Ptr)
               DllCall("RtlMoveMemory", Ptr, pHeapNew, Ptr, pHeap, Ptr, bytesRead)
               DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
               pHeap := pHeapNew
            }
            if !DllCall("Wininet\InternetReadFile", Ptr, hUrl, Ptr, pHeap + bytesRead, UInt, size, UIntP, read)
               break
            bytesRead += read
         }
         DllCall("Wininet\InternetCloseHandle", Ptr, hUrl)
      } until EOF || (attemptCount = 3 && error := "Can't download the file. Unknown error")
   }
   ( hInternet && DllCall("Wininet\InternetCloseHandle", Ptr, hInternet) )
   ( hLib && DllCall("FreeLibrary", Ptr, hLib) )
   if error {
      DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
      throw Exception(error)
   }
   else {
      VarSetCapacity(data, bytesRead, 0)
      DllCall("RtlMoveMemory", Ptr, &data, Ptr, pHeap, Ptr, bytesRead)
   }
   DllCall("HeapFree", Ptr, hHeap, UInt, 0, Ptr, pHeap)
   Return bytesRead
}
tmplinshi
Posts: 1604
Joined: 01 Oct 2013, 14:57

Re: convert InternetFileRead() to unicode

07 Dec 2019, 13:35

Works fine here (OS: Win10). I think that's because your system doesn't support decoding Brotli (br).

Code: Select all

MsgBox % GetContentEncoding("https://www.youtube.com/")           ; br
MsgBox % GetContentEncoding("https://www.autohotkey.com/boards/") ; gzip

GetContentEncoding(Url) {
	whr := ComObjCreate("WinHttp.WinHttpRequest.5.1")
	whr.Open("GET", Url)
	whr.SetRequestHeader("Accept-Encoding", "gzip, deflate, br")
	whr.Send()
	return whr.GetResponseHeader("Content-Encoding")
}
So br should be removed to be more compatible, thanks for testing!
teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

07 Dec 2019, 13:44

Yeah, you are right, on Windows 10 your code works, thanks!
guest3456
Posts: 3454
Joined: 09 Oct 2013, 10:31

Re: convert InternetFileRead() to unicode

07 Dec 2019, 18:23

there was another change needed in InternetFileRead() for unicode:

Code: Select all

;before
VarSetCapacity( V,64 ), VarSetCapacity( V,0 )
;after
VarSetCapacity( V,128 ), VarSetCapacity( V,0 )
the variable wasn't shrinking, probably due to the note in the VarSetCapacity docs

however

it turns out that extra 1 byte in the length in unicode is going to cause me problems with my decryption. it doesnt look like DataFromURL() has that extra byte problem, so i will use that instead. can you guys explain what this gzip and br support means?

@teadrinker also, why the Loop, 1? is it so you can 'break' instead of using goto ?

teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

08 Dec 2019, 09:49

guest3456 wrote: why the Loop, 1? is it so you can 'break' instead of using goto ?
Yes, I don't like using labels inside functions.
guest3456 wrote: can you guys explain what this gzip and br support means?
Some sites can return data in compressed formats, that reduces the amount of data. Gzip, deflate and br (brotli) are compression algorithms.
teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

08 Dec 2019, 10:14

@tmplinshi
It would be great if there was a way to make sure that the system supports a certain algorithm before using it.
tmplinshi
Posts: 1604
Joined: 01 Oct 2013, 14:57

Re: convert InternetFileRead() to unicode

08 Dec 2019, 12:06

@teadrinker Not sure if you can just check the OS version.

Code: Select all

if (A_OSVersion ~= "10\.\d+\.\d+")
	headers := "Accept-Encoding: gzip, deflate, br"
else
	headers := "Accept-Encoding: gzip, deflate"
teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

08 Dec 2019, 12:15

I see, but someone can install this on his Windows 7, and his system will support brotli.
User avatar
jNizM
Posts: 3183
Joined: 30 Sep 2013, 01:33
Contact:

Re: convert InternetFileRead() to unicode

09 Dec 2019, 05:49

I use Bentschi's functions

DownloadToFile

Code: Select all

DownloadToFile(url, filename)
{
    static a := "AutoHotkey/" A_AhkVersion
    if (!(o := FileOpen(filename, "w")) || !DllCall("LoadLibrary", "str", "wininet") || !(h := DllCall("wininet\InternetOpen", "str", a, "uint", 1, "ptr", 0, "ptr", 0, "uint", 0, "ptr")))
        return 0
    c := s := 0
    if (f := DllCall("wininet\InternetOpenUrl", "ptr", h, "str", url, "ptr", 0, "uint", 0, "uint", 0x80003000, "ptr", 0, "ptr"))
    {
        while (DllCall("wininet\InternetQueryDataAvailable", "ptr", f, "uint*", s, "uint", 0, "ptr", 0) && s > 0)
        {
            VarSetCapacity(b, s, 0)
            DllCall("wininet\InternetReadFile", "ptr", f, "ptr", &b, "uint", s, "uint*", r)
            c += r
            o.rawWrite(b, r)
        }
        DllCall("wininet\InternetCloseHandle", "ptr", f)
    }
    DllCall("wininet\InternetCloseHandle", "ptr", h)
    o.close()
    return c
}
DownloadToString

Code: Select all

DownloadToString(url, encoding := "utf-8")
{
    static a := "AutoHotkey/" A_AhkVersion
    if (!DllCall("LoadLibrary", "str", "wininet") || !(h := DllCall("wininet\InternetOpen", "str", a, "uint", 1, "ptr", 0, "ptr", 0, "uint", 0, "ptr")))
        return 0
    c := s := 0, o := ""
    if (f := DllCall("wininet\InternetOpenUrl", "ptr", h, "str", url, "ptr", 0, "uint", 0, "uint", 0x80003000, "ptr", 0, "ptr"))
    {
        while (DllCall("wininet\InternetQueryDataAvailable", "ptr", f, "uint*", s, "uint", 0, "ptr", 0) && s > 0)
        {
            VarSetCapacity(b, s, 0)
            DllCall("wininet\InternetReadFile", "ptr", f, "ptr", &b, "uint", s, "uint*", r)
            o .= StrGet(&b, r >> (encoding = "utf-16" || encoding = "cp1200"), encoding)
        }
        DllCall("wininet\InternetCloseHandle", "ptr", f)
    }
    DllCall("wininet\InternetCloseHandle", "ptr", h)
    return o
}
DownloadBin

Code: Select all

DownloadBin(url, byref buf)
{
    static a := "AutoHotkey/" A_AhkVersion
    if (!DllCall("LoadLibrary", "str", "wininet", "ptr") || !(h := DllCall("wininet\InternetOpen", "ptr", &a, "uint", 1, "ptr", 0, "ptr", 0, "uint", 0, "ptr")))
        return 0
    c := s := 0
    if (f := DllCall("wininet\InternetOpenUrl", "ptr", h, "str", url, "ptr", 0, "uint", 0, "uint", 0x80003000, "ptr", 0, "ptr"))
    {
        while (DllCall("wininet\InternetQueryDataAvailable", "ptr", f, "uint*", s, "uint", 0, "ptr", 0) && s > 0)
        {
            VarSetCapacity(b, c + s, 0)
            if (c > 0)
                DllCall("RtlMoveMemory", "ptr", &b, "ptr", &buf, "ptr", c)
            DllCall("wininet\InternetReadFile", "ptr", f, "ptr", &b + c, "uint", s, "uint*", r)
            c += r
            VarSetCapacity(buf, c, 0)
            if (c > 0)
                DllCall("RtlMoveMemory", "ptr", &buf, "ptr", &b, "ptr", c)
        }
        DllCall("wininet\InternetCloseHandle", "ptr", f)
    }
    DllCall("wininet\InternetCloseHandle", "ptr", h)
    return c
}
[AHK] v2.0.5 | [WIN] 11 Pro (Version 22H2) | [GitHub] Profile
teadrinker
Posts: 4309
Joined: 29 Mar 2015, 09:41
Contact:

Re: convert InternetFileRead() to unicode

09 Dec 2019, 07:23

@jNizM
You could use URLDownloadToFile instead of your DownloadToFile(). DownloadToString() and DownloadBin() are mostly the same, but less reliable than my DataFromUrl(). My function can resume download after loosing connection.

Return to “Ask for Help (v1)”

Who is online

Users browsing this forum: dunnerca, TAC109, wilkster and 124 guests