A script for English Learners that - using InternetExplorer object - finds new words to learn in steemit posts

Post your working scripts, libraries and tools
nilsonrdg
Posts: 15
Joined: 16 Feb 2017, 16:05
Location: Brazil
Contact:

A script for English Learners that - using InternetExplorer object - finds new words to learn in steemit posts

10 Jan 2019, 16:48

This is a script for foreign English learners.

The script automatically search in the steemit posts for texts that can be used to find sentences with new words (new words to learn): less common words.

It keeps a list of words that you've learned (a list indexed and sorted by the frequence which the words appears).

It also allows to post/twitter the sentences.

There's an attempt to detect English language, but it's not perfect. So, sometimes it's needed to repeat the search in the steemit posts.

Code: Select all

#NoEnv
#SingleInstance force

Global GoogleBooksCommonWordsPath
Global TextFilePath
Global RemainingWords
Global TextLanguage
Global SteemitURL
Global DicMostCommonEngPath
Global DictionaryUrl
Global RecentWordsPath
Global PostingURL

PostingURL = https://twitter.com/

RecentWordsPath=%A_ScriptDir%/RecentWords.txt

DictionaryUrl = https://www.google.com/search?q=meaning+of+

TextLanguage = English

SteemitURL = https://steemit.com/created/

GoogleBooksCommonWordsPath = %A_ScriptDir%/google-books-common-words.txt

TextFilePath = %A_ScriptDir%/TextFile.txt



; ----------------- Creating the interface ----------------------

Gui, font, s10, Verdana 

Gui, Add, Text,,
Gui, Add, Button, gGetWebPageData, Get Webpage Data
Gui, Add, Text, , Page URL
Gui, Add, Edit, vPageUrl w200 h80, 
Gui, Add, Text, , Article Title
Gui, Add, Edit, vPageTitle w200 h80,

Gui, Add, Text, ,
Gui, Add, Button, vAbout gAbout, About

Gui, Add, Button, gScriptFolder, Script Folder
Gui, Add, Button, gReload, Reload
Gui, Add, Text, vInf w200 h80, Ready to start!

Gui, Add, Text, ym, Select a Word
Gui, Add, ListBox, gSelectingWords vSelectingWordListBox w220 h400 Sort,

Gui, Add, Button, gTranslate, Translate
Gui, Add, Button, gDictionary, Dictionary
Gui, font, s13, Verdana 
Gui, Add, Text, vProgress w200 h80,
Gui, font, s10, Verdana 

Gui, Add, Text, ym, Preview
Gui, Add, Edit, vPreview w400 h300, 
Gui, Add, Text, , 
Gui, Add, Text, , INSTRUCTIONS:`n`n - To post: select a word and press Control+Alt+P`n`n - To search in dictionary: select a word and press Control+Alt+D
Gui, Show,, SttPscript

Initialize()

return


; -------------- BUTTONS ACTIONS --------------------

Dictionary:
GuiControlGet, WordChoice,,SelectingWordListBox

StringReplace, WordChoice,WordChoice, `r,, All
StringReplace, WordChoice,WordChoice, `n,, All
RegExReplace(WordChoice,"\s","+")

run, %DictionaryUrl%%WordChoice%
return



Translate:
GuiControlGet, WordChoice,,SelectingWordListBox

Sentence := SentenceFilter(WordChoice)

StringReplace, Sentence,Sentence, `r,, All
StringReplace, Sentence,Sentence, `n,, All

run, https://translate.google.com.br/

Clipboard=%Sentence%

WinWait, Google Tradutor, , 10

sleep, 1000
ShowMessage("Click in the area where you want to insert the text.`nPaste it, if this doesn't work.",0)

WinActivateBottom, Google Tradutor

IfWinActive, Google Tradutor
{
keywait, Lbutton, D
send, ^v
}
return



ScriptFolder:
run %A_ScriptDir%
return



Reload:
Reload
return



About:
run, https://twitter.com/hashtag/SttPscript?src=hash
return


GetWebPageData:
GetWebPageData()
return


; Displays the sentence with the selected word
SelectingWords:
GuiControlGet, WordChoice,,SelectingWordListBox
Sentence := SentenceFilter(WordChoice)

StringReplace, Sentence,Sentence, `r,, All
StringReplace, Sentence,Sentence, `n,, All

; Matching only whole word
Pattern = i)\b%WordChoice%\b

StringUpper, WordUpper, WordChoice

Sentence := RegExReplace(Sentence, Pattern,WordUpper)

guicontrol,, Preview,
sleep, 200
guicontrol,, Preview, %Sentence%

return



guiClose:
exitapp
return

; ---------------- END OF BUTTONS ACTIONS




; ----------- Functions ------------------

; This function shows messages on screen
ShowMessage(ScreenMessage,MessageTime)
{
Progress, m2 b fs12 zh0 y200,  %ScreenMessage%

if (MessageTime>0){
sleep, %MessageTime%
}
else {
KeyWait, LButton, D
}

Progress, Off

}




; Removing most common words
Filter()
{
LastWordsFound=

FileRead, MostCommonWordsContent, %GoogleBooksCommonWordsPath%

; Removing digits
MostCommonWordsContent := RegExReplace(MostCommonWordsContent, ".\d+","")

FileRead, TextFileContents, %TextFilePath%


; Removing steemit user names
TextFileContents := RegExReplace(TextFileContents, "@\w+"," ")

; Removing URL similar
TextFileContents := RegExReplace(TextFileContents, "([--:\w?@%&+~#=]*\.[a-z]{2,4}\/{0,2})((?:[?&](?:\w+)=(?:\w+))+|[--:\w?@%&+~#=]+)?","")

TextFileContents := RemovingSymbolsExcludingSpaces(TextFileContents," ")

; Removing line breaks
TextFileContents := RegExReplace(TextFileContents, "$"," ")

; Removing extra spaces
TextFileContents := RegExReplace(TextFileContents, "\s+"," ")

FileRead, RecentWordsContent, %RecentWordsPath%

RecentWordsContent := RegExReplace(RecentWordsContent, "\d+.","")

MostCommonWordsContent=%RecentWordsContent%%MostCommonWordsContent%

RemainingWords := TextFileContents

Loop, parse, MostCommonWordsContent, `n, `r
{
Word :=A_LoopField

; Matching only whole words
Pattern = i)\b%Word%\b

FoundPos := RegExMatch(RemainingWords, Pattern, OriginalWord)

if (FoundPos>0)
{

; Counting Words (counting spaces =+- number of words)
RemainingWords := RegExReplace(RemainingWords,"\s+"," ")
FileTextTemp:= RegExReplace(RemainingWords, A_Space , A_Space, OutputVarCount)
RemainingLength := OutputVarCount + 1

if (RemainingLength<11)
{
Break
}

GuiControl,, Progress, Remaining words: %RemainingLength%

;msgbox %OriginalWord% - %Pattern%

LastWordsFound=%OriginalWord%`n`r%LastWordsFound%
}

RemainingWords := RegExReplace(RemainingWords, Pattern," ")

}

LastXWordsFound=

; Updating recent words displayed list
;msgbox % LastWordsFound
Loop, parse, LastWordsFound, `n, `r
{
CheckWord(A_LoopField,RecentWordsPath)
LastXWordsFound=%A_LoopField% %LastXWordsFound%

if (A_Index=11){
Break
}

}

FillListBox(LastXWordsFound)

}





FillListBox(StringToFillListBox)
{
StringToFillListBox := RegExReplace(StringToFillListBox, "\s",",")

Sort, StringToFillListBox, D,

StringToFillListBox := RegExReplace(StringToFillListBox, ","," ")

StringToFillListBox := RemovingSymbolsIncludingSpaces(StringToFillListBox,"|")

StringToFillListBox := RegExReplace(StringToFillListBox, "\|+","|")

guicontrol,, SelectingWordListBox,|
guicontrol,, SelectingWordListBox, %StringToFillListBox%

}




; This selects a sentence with the chosen word
SentenceFilter(WordChoice)
{
FileRead, FileContents, %TextFilePath%

ChosenSentences=

FileContents := RegExReplace(FileContents, "\.",".`n`r")

Loop, parse, FileContents, `n, `r
{

; Matching only whole word
Pattern = i)\b%WordChoice%\b


FoundPos := RegExMatch(A_LoopField, Pattern)

if (FoundPos>0)
{
ChosenSentences=%A_LoopField%`n`r%ChosenSentences%
}

}

return %ChosenSentences%
}





GetWebPageData()
{

tooltip, Wait... Searching for some post with enough English Words.

; Creates an Internet Explore COM object
ie := ComObjCreate("InternetExplorer.Application")

LinkIndex:=1
Array := []

loop
{

if (LinkIndex=1)
{
; Navigate to a website
ie.navigate(SteemitURL)

; This ensures the webpage completely loads before doing anything else
while ie.ReadyState != 4
	Sleep, 100

; Shows the actual internet explorer window
ie.visible := true

; Getting Article Links List
Links := ie.document.getElementsByClassName("articles__link")

NumberOflIinks:=Links.Length

; Inserting the links in an array
Loop % Links.Length
{
Array[A_Index - 1] := Links[A_Index - 1].href
}

}

PageUrl := Array[LinkIndex]

LinkIndex:=LinkIndex+1

if (LinkIndex>NumberOflIinks){
LinkIndex := 1
}

GuiControl,, PageUrl, %PageUrl% 

ie.navigate(PageUrl)

; This ensures the webpage completely loads before doing anything else
while ie.ReadyState != 4
	Sleep, 100

; Shows the actual internet explorer window
ie.visible := true

FileContents := ie.document.getElementsByClassName("PostFull__body entry-content")[0].innerHTML

FileContents := RegExReplace(FileContents, "<.+?>" , "")
FileContents := RegExReplace(FileContents, "\&nbsp;" , " ")
FileContents := RegExReplace(FileContents, "\s+" , " ")

FileContentsWordsOnly := RemovingSymbolsExcludingSpaces(FileContents," ")

FileContentsWordsOnly := RegExReplace(FileContentsWordsOnly, "\s+"," ")

; Calculating FileContents length (number of words) by the number of spaces
Len001 := StrLen(FileContents)
FileTextTemp := RegExReplace(FileContents, "\s","")
Len002 := StrLen(FileTextTemp)
FileContents_Length := Len001 - Len002

Language := DetectLanguage(FileContents)

if (FileContents_Length>50 and Language=TextLanguage)
{
PageTitle := ie.document.getElementsByTagName("title")[0].innerHTML

PageTitle := RegExReplace(PageTitle, "— Steemit" , "")

GuiControl,, PageTitle, %PageTitle% 

FileContents := ie.document.getElementsByClassName("PostFull__body entry-content")[0].innerHTML

FileContents := RegExReplace(FileContents, "<.+?>" , "")

FileContents := RegExReplace(FileContents, "\&nbsp;" , " ")

fileDelete, %TextFilePath%
sleep, 250
FileAppend, %FileContents%, %TextFilePath%,UTF-8

; Quits out of the application.
ie.quit


tooltip,
sleep, 1000

Filter()

Break
}

; end loop
}

; end function
}




Initialize()
{

if !FileExist(GoogleBooksCommonWordsPath)
{
tooltip, Downloading google-books-common-words.txt...


UrlDownloadToFile, http://norvig.com/google-books-common-words.txt, %GoogleBooksCommonWordsPath%


if (ErrorLevel=1)
{

msgbox, You must save the file "google-books-common-words.txt" in script folder.

MsgBox, 4,, Do you want to go to`nhttp://norvig.com/google-books-common-words.txt`n`rOr you want to find the file by yourself?`n`nPress 'Yes' to go to the page.
	IfMsgBox Yes
	{
	run, http://norvig.com/google-books-common-words.txt
	}
}

tooltip, 
}

if !FileExist(RecentWordsPath)
{
FileAppend,000000 SttPscript`n,%RecentWordsPath%
}

}



; An attempt to detect text language
DetectLanguage(TextToVerify)
{

; Removing steemit user names
TextToVerify := RegExReplace(TextToVerify, "@\w+"," ")

; Removing Url similar
TextToVerify := RegExReplace(TextToVerify, "([--:\w?@%&+~#=]*\.[a-z]{2,4}\/{0,2})((?:[?&](?:\w+)=(?:\w+))+|[--:\w?@%&+~#=]+)?","")


TextToVerify := RemovingSymbolsExcludingSpaces(TextToVerify," ")

TextToVerify := RegExReplace(TextToVerify, "\s+"," ")
TextToVerify := RegExReplace(TextToVerify, "\s", " ", NumberOfWords)

FileRead, DictionaryContents, %GoogleBooksCommonWordsPath%

k:=0
Counter:=0
EnglishWordsMatched=

Loop, parse, DictionaryContents, `n, `r
{

Word := RemovingSymbolsIncludingSpaces(A_LoopField,"")

; Matching only whole words

Pattern = i)\b%Word%\b

if !Word
{
Break
}

FoundPos := RegExMatch(TextToVerify,Pattern)
; Displaying matched words.
if (FoundPos>0) {
tooltip, English words founded: %k% - Current word: %Word%
EnglishWordsMatched = %EnglishWordsMatched% %Word%
}

NumberOfOcurrences:=0
TextToVerify := RegExReplace(TextToVerify, Pattern, " ", NumberOfOcurrences)
k := k + NumberOfOcurrences


WordFrequence := k/NumberOfWords


if (Counter>20000){
Break
}

if (WordFrequence>0.65) {
Break
}

Counter := Counter + 1

}

tooltip,

if (WordFrequence>0.50)
{

DetectionMessage = Searching post with`nenough English Words.`n`nCurrent post:`n`nNumber of Words: %NumberOfWords%`nWordFrequence: %WordFrequence%`nWords matched: %EnglishWordsMatched%

ShowMessage(DetectionMessage,5000)

Language:=TextLanguage
GuiControl,, Inf, Detected Lang: Eng
return, Language
}
else
{
Language=Other
return, Language
}
}




; This function inserts a word in the file (PathFile) and updates his number of occurrences
CheckWord(WordSearch,PathFile)
{
SearchControl:= 0
StringUpper, WordSearch, WordSearch

FileRead, MostCommonWords, %PathFile%

Loop, parse, MostCommonWords, `n, `r ; ALWAYS add `r as ignore char or it may fail
{

Pattern = \s%WordSearch%$

FoundPos := RegExMatch(A_LoopField, Pattern)

if (FoundPos>0)
{
Frequence := RegExReplace(A_LoopField, "[^\d]","")
NewFrequence := Frequence + 1


NewFrequence := FormatingFrequence(NewFrequence)

Line=%Frequence% %WordSearch%
NewLine=%NewFrequence% %WordSearch%
ReplacingWord(Line,NewLine,PathFile)
FoundPos := 0
SearchControl:= 1
}

}

if (SearchControl=0) {
FileAppend, 000000 %WordSearch%`n, %PathFile%
}

}

; This replaces a word in a file
ReplacingWord(Line,NewLine,PathFile)
{
FileRead, FileContent, %PathFile%

FileContent := RegExReplace(FileContent,Line,NewLine)
FileDelete, %PathFile%
sleep, 250
FileAppend, %FileContent%, %PathFile%
}



; This is used to insert k zeros in the word frequence
FormatingFrequence(k)
{

kLength := StrLen(k)

while (kLength<5) {
k = 0%k%
kLength := StrLen(k)
}
return k
}
; ----------- End Functions ------------------




; ------------- Hotkeys -------------------------


; ==> Search selected word in dictionary
^!D::

Clipboard=

send, ^c
sleep, 250
Word := Clipboard

if word
{
word := RegExReplace(word,"\s","+")
run, %DictionaryUrl%%word%
}
else
{
msgbox, You must select something.
}
return


RemovingSymbolsExcludingSpaces(StringToClean,ReplacementString)
{
StringToClean := RegExReplace(StringToClean, "[^a-zA-Z\-\_\s\']",ReplacementString)
Return %StringToClean%
}

RemovingSymbolsIncludingSpaces(StringToClean,ReplacementString)
{
StringToClean := RegExReplace(StringToClean, "[^a-zA-Z\-\_\']",ReplacementString)
Return %StringToClean%
}



; Post
^!P::

Clipboard=

GuiControlGet, PageUrl,,PageUrl

send, ^c
sleep, 250
Word := Clipboard
Word := RegExReplace(Word,"\s$", "")

if word
{
send, ^a
send, ^c
sleep, 250
Sentence := Clipboard

Sentence := RegExReplace(Sentence,"$", "")
;Sentence := RemovingSymbolsExcludingSpaces(Sentence,"")

StringUpper, WordUpper, Word
Sentence := RegExReplace(Sentence,Word, WordUpper)


WordFitoSearch := RegExReplace(Word,"\s+","+")
PathDic = %DictionaryUrl%%WordFitoSearch%

StringUpper, WordT, Word, T

Clipboard = `nWord of the Day => %WordT%: %PathDic%`n`n => "%Sentence%"`n`nFound in steemit by #SttPscript
run, %PostingURL%

}
else
{
msgbox, You must to select a word in a sentence.
}

ShowMessage("Paste the content in the posting area.",0)

return

; ---------- End of HotKeys --------

; This allows to test filter
^!#F::
filter()
return

Return to “Scripts and Functions”

Who is online

Users browsing this forum: mrlegacy83 and 28 guests