Author Topic: Programatically download the latest 64 bit version of Firefox for Windows.  (Read 6356 times)

Offline erosolmi

  • BASIC Developer
  • Posts: 4
Re: Programatically download the latest 64 bit version of Firefox for Windows.
« Reply #45 on: November 01, 2018, 04:23:27 AM »
Here a thinBasic ASYNC version with callback function used to determine download progression.
Used only libraries released with thinBasic official setup

Progress data is shown on a pop up box that will be removed when download is finished.

Maybe a little bit complex but worth to post to share ideas.

Code: thinBasic
  1. uses "Console"
  2. uses "iNet"
  3. uses "OS"
  4.  
  5. printl "Connect to Firefox web site, determine latest Firefox version and download it"
  6.  
  7. '---Check if computer is connected to internet
  8. if INET_GetState Then
  9.   '---Get Firefox web site page
  10.   string Firefox_Info_Page = INET_UrlGetString("https://www.mozilla.org/en-US/firefox/new/")
  11.  
  12.   '---Get Firefox available version
  13.   string Firefox_Available_Version = grab$(Firefox_Info_Page, "data-latest-firefox=""", """")
  14.  
  15.   '---Get Win64 direct download url
  16.   string Firefox_DownLoad_Url_Win64 = grab$(Firefox_Info_Page, "<div id=""other-platforms"">", "</div>")
  17.   Firefox_DownLoad_Url_Win64 = grab$(Firefox_DownLoad_Url_Win64, "<li class=""os_win64"">", "</li>")
  18.   Firefox_DownLoad_Url_Win64 = grab$(Firefox_DownLoad_Url_Win64, "<a href=""", """")
  19.  
  20.   printl "Firefox available version...:", Firefox_Available_Version
  21.   printl "Firefox Download url........:", Firefox_DownLoad_Url_Win64
  22.  
  23.   '---Download
  24.   string sLocalFile = APP_SourcePath + "Firefox Setup " + Firefox_Available_Version + ".exe"
  25.  
  26.   '---Initializes an application's use of the WinINet Internet functions.
  27.   dword hInternet = INET_Internet_Open("Download Fireforx", %INTERNET_OPEN_TYPE_DIRECT)
  28.   IF hInternet = %NULL THEN
  29.     printl "INET_Internet_Open error"
  30.     waitkey
  31.     stop
  32.   END IF
  33.  
  34.   long MyContext = 1
  35.   long hFile = INET_Internet_OpenUrl(hInternet, Firefox_DownLoad_Url_Win64, "", %INTERNET_FLAG_NO_CACHE_WRITE, MyContext)
  36.   IF hFile = %NULL THEN
  37.     printl "INET_Internet_OpenUrl error. Check url: " & Firefox_DownLoad_Url_Win64
  38.     INET_Internet_CloseHandle hInternet
  39.     waitkey
  40.     stop
  41.   END IF
  42.  
  43. '---Set a callback for hInternet events. Driver will automatically call MyStatusFunction on status change
  44.   long bResult = INET_Internet_SetStatusCallBack(hFile, MyStatusFunction)
  45.  
  46. '---Query remote server about the size of the file we are going to download
  47.   long FileSize = INET_Http_QueryInfo(hFile, %HTTP_QUERY_CONTENT_LENGTH)
  48.   printl "Real file size in bytes: " & FileSize
  49.   IF FileSize = %NULL THEN
  50.     printl "INET_Http_QueryInfo(hInternet, %HTTP_QUERY_CONTENT_LENGTH) error. Check url: " & Firefox_DownLoad_Url_Win64
  51.     INET_Internet_CloseHandle hInternet
  52.     INET_Internet_CloseHandle hFile
  53.     waitkey
  54.     stop
  55.   END IF
  56.  
  57.    
  58. '---Ok we can start a loop for file download. This can be automated in a loop or in a timer function
  59. '---Reads the data in %BufferLen bytes chunks
  60.   double T0 = timer
  61.   double T1
  62.  
  63.   '---Setup a %BufferLen bytes buffer
  64.     long  lBufferLen    = 500000 '500k
  65.     string strBuffer    = STRING$(lBufferLen, $SPC)
  66.     dword pstrBuffer    = STRPTR(strBuffer)
  67.     String strData
  68.     dword cbBytesRead  
  69.     DWord TotalBytesRead
  70.     long  lScreenLine   = 8
  71.     long  lScreenPos    = 20
  72.  
  73.   string sSave = Console_SaveScreen(1, 1, 80, 24)
  74.   Console_Box(lScreenPos - 1, lScreenLine - 1, 40, 8, 24, 24, "Downloading ...", 26, %Console_BOX_FLAG_3DOFF)
  75.   DO
  76.     bResult = INET_Internet_ReadFile(hFile, pstrBuffer, len(strBuffer), cbBytesRead)
  77.     IF bResult = 0 OR cbBytesRead = 0 THEN EXIT DO
  78.     IF cbBytesRead = LEN(strBuffer) THEN
  79.       strData &= strBuffer
  80.     ELSE
  81.       strData &= LEFT$(strBuffer, cbBytesRead)
  82.     END IF
  83.     TotalBytesRead += cbBytesRead
  84.     T1 = Timer
  85.    
  86.     PrintAt "Elapsed seconds  : " & LSet$(Format$(T1-T0, "0"), 19)                                         , lScreenPos + 1, lScreenLine + 1, 24
  87.     PrintAt "Bytes read so far: " & LSet$(TotalBytesRead, 19)                                              , lScreenPos + 1, lScreenLine + 2, 24
  88.     PrintAt "KBytes per second: " & LSet$(Format$((TotalBytesRead / Max(T1-T0, 1))/1024, "0.00"), 19)      , lScreenPos + 1, lScreenLine + 3, 24
  89.     PrintAt "Estimated seconds: " & LSet$(Format$((FileSize * (T1-T0))/TotalBytesRead, "0"), 19)           , lScreenPos + 1, lScreenLine + 4, 24
  90.     PrintAt "Seconds to go    : " & LSet$(Format$(((FileSize * (T1-T0))/TotalBytesRead)-(T1-T0), "0"), 19) , lScreenPos + 1, lScreenLine + 5, 24
  91.    
  92.     Console_ProgressBar(1, lScreenPos + 1, lScreenLine + 7, 40-1, 27, 1, FileSize, TotalBytesRead)
  93.   LOOP
  94.  
  95.   Console_RestoreScreen(1, 1, 80, 24, sSave)
  96.  
  97. '---Closes all used handles
  98.   INET_Internet_CloseHandle hInternet
  99.   INET_Internet_CloseHandle hFile
  100.  
  101.  
  102.   printl "Saving file: " & sLocalFile
  103.   Save_File(sLocalFile, strData)
  104.  
  105.   printl "File saved."
  106.   printl
  107.   printl "Press ESC to end or any other key within 10 secs to execute setup"
  108.  
  109.  
  110.   '---Ask if execute setup
  111.   string sUserReply = WaitKey(10)
  112.   if sUserReply <> "[ESC]" and sUserReply <> "[TIMEOUT]" Then
  113.     printl "Executing " + sLocalFile
  114.     OS_Shell(sLocalFile, %OS_WNDSTYLE_NORMAL, %OS_SHELL_SYNC)
  115.   Else
  116.     printl "No execution. Execute manually if needed."
  117.   end If
  118.  
  119. Else
  120.   printl "! it seems you are not connected to internet." in %CCOLOR_FLIGHTRED
  121. end If
  122.  
  123. printl "---All done, press a key to end or wait 5 secs---"
  124. WaitKey(5)
  125.  
  126.  
  127. '------------------------------------------------------------------
  128. ' CallBack function automatically called by Internet driver when
  129. ' status change for a specific Internet handle to which a callback
  130. ' function has been installed.
  131. ' Parameters are mandatory and are defined my Microsoft
  132. ' callback template. More info at: http://msdn.microsoft.com/en-us/library/aa385121(VS.85).aspx
  133. '------------------------------------------------------------------
  134. callback function MyStatusFunction( _
  135.                 byval hInternet                 as long , _   '---Mandatory in thinBasic and in API
  136.                 byval dwContext                 as long , _   '---Mandatory in thinBasic and in API
  137.                 byval dwInternetStatus          as long , _   '---Mandatory in thinBasic and in API
  138.                 byval lpvStatusInformation      as long , _   '---Optional in thinBasic, mandatory in API
  139.                 byval dwStatusInformationLength as long   _   '---Optional in thinBasic, mandatory in API
  140.              ) as long
  141. '------------------------------------------------------------------
  142.   dim BytesReceived     as dword at 0
  143.   static TotBytes       as dword
  144.  
  145.   static MoreStatusLine as long = 20
  146.      
  147.   'printat "Data from " & function_name & " callback"          , 1, 17
  148.   'printat "  Internet handle: " & hInternet & "  Internet dwContext: " & dwContext, 1, 18
  149.  
  150.   select case dwInternetStatus
  151.     case %INTERNET_STATUS_RECEIVING_RESPONSE
  152.  
  153.     case %INTERNET_STATUS_RESPONSE_RECEIVED
  154.       '---  Successfully received a response from the server.
  155.       '---  The lpvStatusInformation parameter points to a DWORD value that contains the number, in bytes, received.
  156.       setat(BytesReceived, lpvStatusInformation)
  157.       TotBytes += BytesReceived
  158.  
  159.       '---ATTENTION: TotalBytes received can differ from total bytes of the file to be downloaded
  160.  
  161.       'printat "  Receiving data (" & dwInternetStatus & " " & INET_Internet_StatusGetDescription(dwInternetStatus) & ") TotBytes: " & TotBytes, 1, 19
  162.  
  163.     case else
  164.       '---In case of other status, just decode it at the bottom.
  165.       'printat "  (" & dwInternetStatus & " " & INET_Internet_StatusGetDescription(dwInternetStatus) & ")", 1, MoreStatusLine
  166.       incr MoreStatusLine
  167.  
  168.   end select            
  169.  
  170.  
  171. end function
  172.  
  173.  

« Last Edit: November 01, 2018, 08:31:50 AM by John »

Offline AIR

  • BASIC Developer
  • Posts: 539
Re: Programatically download the latest 64 bit version of Firefox for Windows.
« Reply #46 on: November 01, 2018, 09:47:00 AM »
Very Cool, Eros!

Question:  Is your Console_Box constructed via WinAPI calls, or are you using a 3rd party library?

AIR.

Offline erosolmi

  • BASIC Developer
  • Posts: 4
Re: Programatically download the latest 64 bit version of Firefox for Windows.
« Reply #47 on: November 01, 2018, 10:31:36 AM »
All developed by me with only Windows console api:
WriteConsoleOutputCharacter
FillConsoleOutputAttribute
ReadConsoleOutputCharacter

Console module was one of the first module I develop in 2004 when I started to work on thinBasic
« Last Edit: November 01, 2018, 11:09:06 AM by erosolmi »

Offline AIR

  • BASIC Developer
  • Posts: 539
Re: Programatically download the latest 64 bit version of Firefox for Windows.
« Reply #48 on: December 12, 2018, 12:19:18 AM »
Since I'm working on updating MBC, here's an MBC version:

Code: Text
  1. $MODULE "curl.inc"
  2.  
  3. CLS
  4. Dim HTML$,dl_version$,dl_link$,fname$, match as REGEX
  5.  
  6. HTML$ = downloadPage$("https://www.mozilla.org/en-US/firefox/new")
  7.  
  8. if bcxregex(HTML$,"data-latest-firefox=.([0-9]+.[0-9]+\.?[0-9]?)",&match) then dl_version$ = match.results[1]
  9. if bcxregex(HTML$,".+(http.+ssl[&a-z;=-]+win64[&a-z;=-]+)",&match) then dl_link$ = match.results[1]
  10.  
  11. Print E"Firefox Download Challenge (MBC Version) by AIR.\n"
  12. Print E"Downloading Latest 64Bit Firefox (",dl_version,") for Windows.\n"
  13.  
  14. downloadFile(dl_link, "Firefox Setup-" & dl_version$ & ".exe")


Offline John

  • Forum Support / SB Dev
  • Posts: 2416
    • ScriptBasic Open Source Project
Re: Programatically download the latest 64 bit version of Firefox for Windows.
« Reply #49 on: December 12, 2018, 12:43:57 AM »
Old challenges never die, they only get more interesting.

Curious. Can you time your MBC and SB Firefox download challenges for a reference?
« Last Edit: December 12, 2018, 12:51:35 AM by John »

Offline John

  • Forum Support / SB Dev
  • Posts: 2416
    • ScriptBasic Open Source Project
I thought I would mention that one should reinitialize curl between commits. Curl was sending a previous POST buffer with a following GET.

Offline AIR

  • BASIC Developer
  • Posts: 539
Go version:

Code: Go
  1. package main
  2.  
  3. import (
  4.         "fmt"
  5.         "io"
  6.         "io/ioutil"
  7.         "net/http"
  8.         "os"
  9.         "regexp"
  10.         "strings"
  11.  
  12.         "github.com/dustin/go-humanize"
  13. )
  14.  
  15. // WriteCounter counts the number of bytes written to it. It implements to the io.Writer
  16. // interface and we can pass this into io.TeeReader() which will report progress on each
  17. // write cycle.
  18. type WriteCounter struct {
  19.         Total    uint64
  20.         Filesize uint64
  21. }
  22.  
  23. func main() {
  24.         clear()
  25.         fmt.Printf("Firefox Download Challenge (GO Version) by AIR.\n\n")
  26.         url := "https://mozilla.org/firefox/new"
  27.  
  28.         // Retrieve the data from the url
  29.         resp, err := http.Get(url)
  30.         chkError(err)
  31.  
  32.         // Auto close connection
  33.         defer resp.Body.Close()
  34.  
  35.         // reads html as a slice (array) of bytes
  36.         html, err := ioutil.ReadAll(resp.Body)
  37.         chkError(err)
  38.  
  39.         // regex to extract current version
  40.         re, err := regexp.Compile(`data-latest-firefox="([0-9]+\.[0-9]+?\.[0-9]+)"`)
  41.         chkError(err)
  42.  
  43.         // Regex to extract the download link
  44.         re2, err := regexp.Compile(`.+href="(.+latest-ssl.+os=win64.+US).+`)
  45.         chkError(err)
  46.  
  47.         // Perform search for version and download link
  48.         ffVersion := re.FindStringSubmatch(string(html))
  49.         ffLink := re2.FindStringSubmatch(string(html))
  50.  
  51.         // Download the Windows installer
  52.         err = DownloadFile("Firefox Setup "+ffVersion[1]+".exe", ffLink[1])
  53.         chkError(err)
  54.  
  55.         fmt.Println("\nDownload Complete.")
  56. }
  57.  
  58. func chkError(e error) {
  59.         if e != nil {
  60.                 panic(e)
  61.         }
  62. }
  63.  
  64. // Clears screen (Linux/macOS)
  65. func clear() {
  66.         fmt.Print("\033[2J\033[H")
  67. }
  68.  
  69. // DownloadFile will download a url to a local file. It's efficient because it will
  70. // write as it downloads and not load the whole file into memory. We pass an io.TeeReader
  71. // into Copy() to report progress on the download.
  72. func DownloadFile(filepath string, url string) error {
  73.  
  74.         // Create the file
  75.         out, err := os.Create(filepath)
  76.         if err != nil {
  77.                 return err
  78.         }
  79.         defer out.Close()
  80.  
  81.         // Get the data
  82.         resp, err := http.Get(url)
  83.         if err != nil {
  84.                 return err
  85.         }
  86.         defer resp.Body.Close()
  87.  
  88.         // Create our progress reporter (with reported Filesize set) and pass it to be used alongside our writer
  89.         counter := &WriteCounter{Filesize: uint64(resp.ContentLength)}
  90.  
  91.         _, err = io.Copy(out, io.TeeReader(resp.Body, counter))
  92.         if err != nil {
  93.                 return err
  94.         }
  95.  
  96.         // The progress outputs on a single line so print a new line once it's finished downloading
  97.         fmt.Print("\n")
  98.  
  99.         return nil
  100. }
  101.  
  102. // Callback used by WriteCounter
  103. func (wc *WriteCounter) Write(p []byte) (int, error) {
  104.         n := len(p)
  105.         wc.Total += uint64(n)
  106.  
  107.         wc.PrintProgress()
  108.         return n, nil
  109. }
  110.  
  111. // PrintProgress displays the download status
  112. func (wc WriteCounter) PrintProgress() {
  113.         // Clear the line by using a character return to go back to the start and remove
  114.         // the remaining characters by filling it with spaces
  115.         fmt.Printf("\r%s", strings.Repeat(" ", 30))
  116.  
  117.         // Return again and print current status of download
  118.         // We use the humanize package to print the status in a human-readable way (e.g. 10 MB)
  119.         fmt.Printf("\rDownloaded %s of %s", humanize.Bytes(wc.Total), humanize.Bytes(wc.Filesize))
  120.  
  121. }
  122.  

Output:
Firefox Download Challenge (GO Version) by AIR.

Downloaded 46 MB of 46 MB

Download Complete.


AIR.