Programatically download the latest 64 bit version of Firefox for Windows.

[erosolmi]

Here a thinBasic ASYNC version with callback function used to determine download progression.
Used only libraries released with thinBasic official setup

Progress data is shown on a pop up box that will be removed when download is finished.

Maybe a little bit complex but worth to post to share ideas.

Code: thinBasic

1. uses "Console"
2. uses "iNet"
3. uses "OS"
4.  
5. printl "Connect to Firefox web site, determine latest Firefox version and download it"
6.  
7. '---Check if computer is connected to internet
8. if INET_GetState Then
9.   '---Get Firefox web site page
10.   string Firefox_Info_Page = INET_UrlGetString("https://www.mozilla.org/en-US/firefox/new/")
11.  
12.   '---Get Firefox available version
13.   string Firefox_Available_Version = grab$(Firefox_Info_Page, "data-latest-firefox=""", """")
14.  
15.   '---Get Win64 direct download url
16.   string Firefox_DownLoad_Url_Win64 = grab$(Firefox_Info_Page, "<div id=""other-platforms"">", "</div>")
17.   Firefox_DownLoad_Url_Win64 = grab$(Firefox_DownLoad_Url_Win64, "<li class=""os_win64"">", "</li>")
18.   Firefox_DownLoad_Url_Win64 = grab$(Firefox_DownLoad_Url_Win64, "<a href=""", """")
19.  
20.   printl "Firefox available version...:", Firefox_Available_Version
21.   printl "Firefox Download url........:", Firefox_DownLoad_Url_Win64
22.  
23.   '---Download
24.   string sLocalFile = APP_SourcePath + "Firefox Setup " + Firefox_Available_Version + ".exe"
25.  
26.   '---Initializes an application's use of the WinINet Internet functions.
27.   dword hInternet = INET_Internet_Open("Download Fireforx", %INTERNET_OPEN_TYPE_DIRECT)
28.   IF hInternet = %NULL THEN
29.     printl "INET_Internet_Open error"
30.     waitkey
31.     stop
32.   END IF
33.  
34.   long MyContext = 1
35.   long hFile = INET_Internet_OpenUrl(hInternet, Firefox_DownLoad_Url_Win64, "", %INTERNET_FLAG_NO_CACHE_WRITE, MyContext)
36.   IF hFile = %NULL THEN
37.     printl "INET_Internet_OpenUrl error. Check url: " & Firefox_DownLoad_Url_Win64
38.     INET_Internet_CloseHandle hInternet
39.     waitkey
40.     stop
41.   END IF
42.  
43. '---Set a callback for hInternet events. Driver will automatically call MyStatusFunction on status change
44.   long bResult = INET_Internet_SetStatusCallBack(hFile, MyStatusFunction)
45.  
46. '---Query remote server about the size of the file we are going to download
47.   long FileSize = INET_Http_QueryInfo(hFile, %HTTP_QUERY_CONTENT_LENGTH)
48.   printl "Real file size in bytes: " & FileSize
49.   IF FileSize = %NULL THEN
50.     printl "INET_Http_QueryInfo(hInternet, %HTTP_QUERY_CONTENT_LENGTH) error. Check url: " & Firefox_DownLoad_Url_Win64
51.     INET_Internet_CloseHandle hInternet
52.     INET_Internet_CloseHandle hFile
53.     waitkey
54.     stop
55.   END IF
56.  
57.    
58. '---Ok we can start a loop for file download. This can be automated in a loop or in a timer function
59. '---Reads the data in %BufferLen bytes chunks
60.   double T0 = timer
61.   double T1
62.  
63.   '---Setup a %BufferLen bytes buffer
64.     long  lBufferLen    = 500000 '500k
65.     string strBuffer    = STRING$(lBufferLen, $SPC)
66.     dword pstrBuffer    = STRPTR(strBuffer)
67.     String strData
68.     dword cbBytesRead  
69.     DWord TotalBytesRead
70.     long  lScreenLine   = 8
71.     long  lScreenPos    = 20
72.  
73.   string sSave = Console_SaveScreen(1, 1, 80, 24)
74.   Console_Box(lScreenPos - 1, lScreenLine - 1, 40, 8, 24, 24, "Downloading ...", 26, %Console_BOX_FLAG_3DOFF)
75.   DO
76.     bResult = INET_Internet_ReadFile(hFile, pstrBuffer, len(strBuffer), cbBytesRead)
77.     IF bResult = 0 OR cbBytesRead = 0 THEN EXIT DO
78.     IF cbBytesRead = LEN(strBuffer) THEN
79.       strData &= strBuffer
80.     ELSE
81.       strData &= LEFT$(strBuffer, cbBytesRead)
82.     END IF
83.     TotalBytesRead += cbBytesRead
84.     T1 = Timer
85.    
86.     PrintAt "Elapsed seconds  : " & LSet$(Format$(T1-T0, "0"), 19)                                         , lScreenPos + 1, lScreenLine + 1, 24
87.     PrintAt "Bytes read so far: " & LSet$(TotalBytesRead, 19)                                              , lScreenPos + 1, lScreenLine + 2, 24
88.     PrintAt "KBytes per second: " & LSet$(Format$((TotalBytesRead / Max(T1-T0, 1))/1024, "0.00"), 19)      , lScreenPos + 1, lScreenLine + 3, 24
89.     PrintAt "Estimated seconds: " & LSet$(Format$((FileSize * (T1-T0))/TotalBytesRead, "0"), 19)           , lScreenPos + 1, lScreenLine + 4, 24
90.     PrintAt "Seconds to go    : " & LSet$(Format$(((FileSize * (T1-T0))/TotalBytesRead)-(T1-T0), "0"), 19) , lScreenPos + 1, lScreenLine + 5, 24
91.    
92.     Console_ProgressBar(1, lScreenPos + 1, lScreenLine + 7, 40-1, 27, 1, FileSize, TotalBytesRead)
93.   LOOP
94.  
95.   Console_RestoreScreen(1, 1, 80, 24, sSave)
96.  
97. '---Closes all used handles
98.   INET_Internet_CloseHandle hInternet
99.   INET_Internet_CloseHandle hFile
100.  
101.  
102.   printl "Saving file: " & sLocalFile
103.   Save_File(sLocalFile, strData)
104.  
105.   printl "File saved."
106.   printl
107.   printl "Press ESC to end or any other key within 10 secs to execute setup"
108.  
109.  
110.   '---Ask if execute setup
111.   string sUserReply = WaitKey(10)
112.   if sUserReply <> "[ESC]" and sUserReply <> "[TIMEOUT]" Then
113.     printl "Executing " + sLocalFile
114.     OS_Shell(sLocalFile, %OS_WNDSTYLE_NORMAL, %OS_SHELL_SYNC)
115.   Else
116.     printl "No execution. Execute manually if needed."
117.   end If
118.  
119. Else
120.   printl "! it seems you are not connected to internet." in %CCOLOR_FLIGHTRED
121. end If
122.  
123. printl "---All done, press a key to end or wait 5 secs---"
124. WaitKey(5)
125.  
126.  
127. '------------------------------------------------------------------
128. ' CallBack function automatically called by Internet driver when
129. ' status change for a specific Internet handle to which a callback
130. ' function has been installed.
131. ' Parameters are mandatory and are defined my Microsoft
132. ' callback template. More info at: http://msdn.microsoft.com/en-us/library/aa385121(VS.85).aspx
133. '------------------------------------------------------------------
134. callback function MyStatusFunction( _
135.                 byval hInternet                 as long , _   '---Mandatory in thinBasic and in API
136.                 byval dwContext                 as long , _   '---Mandatory in thinBasic and in API
137.                 byval dwInternetStatus          as long , _   '---Mandatory in thinBasic and in API
138.                 byval lpvStatusInformation      as long , _   '---Optional in thinBasic, mandatory in API
139.                 byval dwStatusInformationLength as long   _   '---Optional in thinBasic, mandatory in API
140.              ) as long
141. '------------------------------------------------------------------
142.   dim BytesReceived     as dword at 0
143.   static TotBytes       as dword
144.  
145.   static MoreStatusLine as long = 20
146.      
147.   'printat "Data from " & function_name & " callback"          , 1, 17
148.   'printat "  Internet handle: " & hInternet & "  Internet dwContext: " & dwContext, 1, 18
149.  
150.   select case dwInternetStatus
151.     case %INTERNET_STATUS_RECEIVING_RESPONSE
152.  
153.     case %INTERNET_STATUS_RESPONSE_RECEIVED
154.       '---  Successfully received a response from the server.
155.       '---  The lpvStatusInformation parameter points to a DWORD value that contains the number, in bytes, received.
156.       setat(BytesReceived, lpvStatusInformation)
157.       TotBytes += BytesReceived
158.  
159.       '---ATTENTION: TotalBytes received can differ from total bytes of the file to be downloaded
160.  
161.       'printat "  Receiving data (" & dwInternetStatus & " " & INET_Internet_StatusGetDescription(dwInternetStatus) & ") TotBytes: " & TotBytes, 1, 19
162.  
163.     case else
164.       '---In case of other status, just decode it at the bottom.
165.       'printat "  (" & dwInternetStatus & " " & INET_Internet_StatusGetDescription(dwInternetStatus) & ")", 1, MoreStatusLine
166.       incr MoreStatusLine
167.  
168.   end select            
169.  
170.  
171. end function
172.  
173.  

[AIR]

Very Cool, Eros!

Question:  Is your Console_Box constructed via WinAPI calls, or are you using a 3rd party library?

AIR.

[erosolmi]

All developed by me with only Windows console api:
WriteConsoleOutputCharacter
FillConsoleOutputAttribute
ReadConsoleOutputCharacter

Console module was one of the first module I develop in 2004 when I started to work on thinBasic

[AIR]

Since I'm working on updating MBC, here's an MBC version:

Code: Text

1. $MODULE "curl.inc"
2.  
3. CLS
4. Dim HTML$,dl_version$,dl_link$,fname$, match as REGEX
5.  
6. HTML$ = downloadPage$("https://www.mozilla.org/en-US/firefox/new")
7.  
8. if bcxregex(HTML$,"data-latest-firefox=.([0-9]+.[0-9]+\.?[0-9]?)",&match) then dl_version$ = match.results[1]
9. if bcxregex(HTML$,".+(http.+ssl[&a-z;=-]+win64[&a-z;=-]+)",&match) then dl_link$ = match.results[1]
10.  
11. Print E"Firefox Download Challenge (MBC Version) by AIR.\n"
12. Print E"Downloading Latest 64Bit Firefox (",dl_version,") for Windows.\n"
13.  
14. downloadFile(dl_link, "Firefox Setup-" & dl_version$ & ".exe")

[John]

Old challenges never die, they only get more interesting.

Curious. Can you time your MBC and SB Firefox download challenges for a reference?

[John]

I thought I would mention that one should reinitialize curl between commits. Curl was sending a previous POST buffer with a following GET.

[AIR]

Go version:

Code: Go

1. package main
2.  
3. import (
4.         "fmt"
5.         "io"
6.         "io/ioutil"
7.         "net/http"
8.         "os"
9.         "regexp"
10.         "strings"
11.  
12.         "github.com/dustin/go-humanize"
13. )
14.  
15. // WriteCounter counts the number of bytes written to it. It implements to the io.Writer
16. // interface and we can pass this into io.TeeReader() which will report progress on each
17. // write cycle.
18. type WriteCounter struct {
19.         Total    uint64
20.         Filesize uint64
21. }
22.  
23. func main() {
24.         clear()
25.         fmt.Printf("Firefox Download Challenge (GO Version) by AIR.\n\n")
26.         url := "https://mozilla.org/firefox/new"
27.  
28.         // Retrieve the data from the url
29.         resp, err := http.Get(url)
30.         chkError(err)
31.  
32.         // Auto close connection
33.         defer resp.Body.Close()
34.  
35.         // reads html as a slice (array) of bytes
36.         html, err := ioutil.ReadAll(resp.Body)
37.         chkError(err)
38.  
39.         // regex to extract current version
40.         re, err := regexp.Compile(`data-latest-firefox="([0-9]+\.[0-9]+?\.[0-9]+)"`)
41.         chkError(err)
42.  
43.         // Regex to extract the download link
44.         re2, err := regexp.Compile(`.+href="(.+latest-ssl.+os=win64.+US).+`)
45.         chkError(err)
46.  
47.         // Perform search for version and download link
48.         ffVersion := re.FindStringSubmatch(string(html))
49.         ffLink := re2.FindStringSubmatch(string(html))
50.  
51.         // Download the Windows installer
52.         err = DownloadFile("Firefox Setup "+ffVersion[1]+".exe", ffLink[1])
53.         chkError(err)
54.  
55.         fmt.Println("\nDownload Complete.")
56. }
57.  
58. func chkError(e error) {
59.         if e != nil {
60.                 panic(e)
61.         }
62. }
63.  
64. // Clears screen (Linux/macOS)
65. func clear() {
66.         fmt.Print("\033[2J\033[H")
67. }
68.  
69. // DownloadFile will download a url to a local file. It's efficient because it will
70. // write as it downloads and not load the whole file into memory. We pass an io.TeeReader
71. // into Copy() to report progress on the download.
72. func DownloadFile(filepath string, url string) error {
73.  
74.         // Create the file
75.         out, err := os.Create(filepath)
76.         if err != nil {
77.                 return err
78.         }
79.         defer out.Close()
80.  
81.         // Get the data
82.         resp, err := http.Get(url)
83.         if err != nil {
84.                 return err
85.         }
86.         defer resp.Body.Close()
87.  
88.         // Create our progress reporter (with reported Filesize set) and pass it to be used alongside our writer
89.         counter := &WriteCounter{Filesize: uint64(resp.ContentLength)}
90.  
91.         _, err = io.Copy(out, io.TeeReader(resp.Body, counter))
92.         if err != nil {
93.                 return err
94.         }
95.  
96.         // The progress outputs on a single line so print a new line once it's finished downloading
97.         fmt.Print("\n")
98.  
99.         return nil
100. }
101.  
102. // Callback used by WriteCounter
103. func (wc *WriteCounter) Write(p []byte) (int, error) {
104.         n := len(p)
105.         wc.Total += uint64(n)
106.  
107.         wc.PrintProgress()
108.         return n, nil
109. }
110.  
111. // PrintProgress displays the download status
112. func (wc WriteCounter) PrintProgress() {
113.         // Clear the line by using a character return to go back to the start and remove
114.         // the remaining characters by filling it with spaces
115.         fmt.Printf("\r%s", strings.Repeat(" ", 30))
116.  
117.         // Return again and print current status of download
118.         // We use the humanize package to print the status in a human-readable way (e.g. 10 MB)
119.         fmt.Printf("\rDownloaded %s of %s", humanize.Bytes(wc.Total), humanize.Bytes(wc.Filesize))
120.  
121. }
122.  

Output:
Firefox Download Challenge (GO Version) by AIR.

Downloaded 46 MB of 46 MB

Download Complete.

AIR.