diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 66bf715c..c9105abb 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -89,4 +89,5 @@ dependencies { implementation ("com.google.code.gson:gson:2.11.0") implementation ("io.realm.kotlin:library-base:2.1.0") implementation ("org.jsoup:jsoup:1.18.1") + implementation ("org.apache.commons:commons-text:1.10.0") } diff --git a/app/src/main/kotlin/rasel/lunar/launcher/home/LauncherHome.kt b/app/src/main/kotlin/rasel/lunar/launcher/home/LauncherHome.kt index b05888d2..69e052c3 100644 --- a/app/src/main/kotlin/rasel/lunar/launcher/home/LauncherHome.kt +++ b/app/src/main/kotlin/rasel/lunar/launcher/home/LauncherHome.kt @@ -50,10 +50,13 @@ import androidx.work.ExistingPeriodicWorkPolicy import androidx.work.PeriodicWorkRequestBuilder import androidx.work.WorkManager import com.google.gson.Gson -import kotlinx.coroutines.MainScope -import kotlinx.coroutines.launch +import org.apache.commons.text.StringEscapeUtils +import org.json.JSONArray +import org.json.JSONObject import org.jsoup.Jsoup import org.jsoup.nodes.Document +import org.xmlpull.v1.XmlPullParser +import org.xmlpull.v1.XmlPullParserFactory import rasel.lunar.launcher.LauncherActivity.Companion.TEST_PAG import rasel.lunar.launcher.LauncherActivity.Companion.lActivity import rasel.lunar.launcher.R @@ -69,23 +72,24 @@ import rasel.lunar.launcher.helpers.UniUtils.Companion.biometricPromptInfo import rasel.lunar.launcher.helpers.UniUtils.Companion.canAuthenticate import rasel.lunar.launcher.helpers.UniUtils.Companion.expandNotificationPanel import rasel.lunar.launcher.helpers.UniUtils.Companion.lockMethod -import rasel.lunar.launcher.home.LauncherHome.Companion.lastedFinishedPageUrl -import rasel.lunar.launcher.home.LauncherHome.Companion.listItem -import rasel.lunar.launcher.home.LauncherHome.Companion.listTags import rasel.lunar.launcher.home.LauncherHome.Companion.refreshCalls import rasel.lunar.launcher.home.LauncherHome.Companion.refreshSms +import rasel.lunar.launcher.home.weather.JsonParser import rasel.lunar.launcher.home.weather.WeatherExecutor import rasel.lunar.launcher.qaccess.QuickAccess import rasel.lunar.launcher.settings.SettingsActivity import rasel.lunar.launcher.todos.MissedCallsAdapter -import rasel.lunar.launcher.todos.RssItemAdapter -import rasel.lunar.launcher.todos.RssTagAdapter import rasel.lunar.launcher.todos.SmsLogsAdapter import rasel.lunar.launcher.utils.BLog import rasel.lunar.launcher.utils.MissedCallGetter import rasel.lunar.launcher.utils.RecentSmsGetter import rasel.lunar.launcher.utils.RecentSmsLog +import rasel.lunar.launcher.utils.RssList import rasel.lunar.launcher.utils.SimpleFingerGestures +import java.io.ByteArrayInputStream +import java.io.InputStream +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets import java.text.SimpleDateFormat import java.util.Calendar import java.util.Date @@ -220,60 +224,15 @@ internal class LauncherHome : Fragment() { chooseAdpater() } binding.otherCheck.setOnClickListener { - BLog.LOGE("binding.otherCheck before ThreadRun") + doWebPare(RssList.youtubeUrls.get(0)) - binding.searcher01.webViewClient = object : WebViewClient() { - override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) { - BLog.LOGE("binding.otherCheck searcher01 in onPageStarted ${url}") - super.onPageStarted(view, url, favicon) - } + } - override fun onReceivedSslError( - view: WebView?, - handler: SslErrorHandler?, - error: SslError? - ) { - handler?.proceed() - } - - - override fun onPageFinished(view: WebView?, url: String?) { - super.onPageFinished(view, url) - lastedFinishedPageUrl = url ?: "" - BLog.LOGE("binding.otherCheck searcher01 in onPageFinished ${url}") - view?.evaluateJavascript("function getAll() {\n" + - " MyJavaScriptInterface.sendValueFromHtml(document.getElementsByTagName('html')[0].innerHTML)" + - " };getAll()") { result -> - (result as? String)?.let { - - } - } - } - } - WebView.setWebContentsDebuggingEnabled(false) - binding.searcher01.apply { - this.addJavascriptInterface(MyJavaScriptInterface(this),"MyJavaScriptInterface") - setBackgroundColor(Color.WHITE) // 백그라운드 색상 설정 - setLayerType(View.LAYER_TYPE_SOFTWARE, null) // 랜더링 이슈 해결 - try { - settings.apply { - javaScriptEnabled = true // 자바스크립트 사용 가능하도록 설정 - loadWithOverviewMode = true // 전체 웹페이지를 화면에 맞게 로드 - useWideViewPort = true // 화면에 맞게 페이지 확대/축소 - domStorageEnabled = true // DOM 저장소 사용 가능하도록 설정 - setSupportMultipleWindows(true) - javaScriptCanOpenWindowsAutomatically = true // 팝업창 차단 해제 - cacheMode = WebSettings.LOAD_CACHE_ELSE_NETWORK - textZoom = 100 // system 에 의한 글꼴 변형 방지 - defaultTextEncodingName = "UTF-8" // 인코딩 설정 - allowContentAccess = true // 웹뷰를 통해 content url에 접근할지 여부 - layoutAlgorithm = WebSettings.LayoutAlgorithm.TEXT_AUTOSIZING // 웹페이지의 레이아웃을 화면에 맞게 자동으로 조정 - } - } catch (ignore: NoSuchMethodError) {} -//TEST_PAG - loadUrl(TEST_PAG) // 웹페이지 연결 - } + binding.otherCheck.setOnLongClickListener { + listItem.clear() + doWebPare(TEST_PAG) + true } binding.summaryChoose.setOnCheckedChangeListener { group, checkedId -> chooseAdpater() @@ -284,6 +243,74 @@ internal class LauncherHome : Fragment() { // BLog.LOGE("intent >>> ${i.action}") } + fun doWebPare(url : String) { + BLog.LOGE("binding.otherCheck before ThreadRun") + binding.searcher01.webViewClient = object : WebViewClient() { + override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) { + BLog.LOGE("binding.otherCheck searcher01 in onPageStarted ${url}") + super.onPageStarted(view, url, favicon) + } + + override fun onReceivedSslError( + view: WebView?, + handler: SslErrorHandler?, + error: SslError? + ) { + handler?.proceed() + } + + + override fun onPageFinished(view: WebView?, url: String?) { + super.onPageFinished(view, url) + lastedFinishedPageUrl = url ?: "" + BLog.LOGE("binding.otherCheck searcher01 in onPageFinished ${url}") + + if (url?.contains("youtube", false) == true) { + view?.evaluateJavascript( + "function getAll() {\n" + + " MyJavaScriptInterface.sendValueFromHtml(document.getElementsByTagName('html')[0].innerHTML)" + + " };getAll()" + ) { result -> + (result as? String)?.let { + + } + } + } else { + view?.evaluateJavascript( + "function getAll() {\n" + + " MyJavaScriptInterface.sendValueFromHtml(document.getElementsByTagName('html')[0].innerHTML)" + + " };getAll()" + ) { result -> + (result as? String)?.let { + + } + } + } + } + } + WebView.setWebContentsDebuggingEnabled(false) + binding.searcher01.apply { + this.addJavascriptInterface(MyJavaScriptInterface(this),"MyJavaScriptInterface") + setBackgroundColor(Color.WHITE) // 백그라운드 색상 설정 + setLayerType(View.LAYER_TYPE_SOFTWARE, null) // 랜더링 이슈 해결 + try { + settings.apply { + javaScriptEnabled = true // 자바스크립트 사용 가능하도록 설정 + loadWithOverviewMode = true // 전체 웹페이지를 화면에 맞게 로드 + useWideViewPort = true // 화면에 맞게 페이지 확대/축소 + domStorageEnabled = true // DOM 저장소 사용 가능하도록 설정 + setSupportMultipleWindows(true) + javaScriptCanOpenWindowsAutomatically = true // 팝업창 차단 해제 + cacheMode = WebSettings.LOAD_CACHE_ELSE_NETWORK + textZoom = 100 // system 에 의한 글꼴 변형 방지 + defaultTextEncodingName = "UTF-8" // 인코딩 설정 + allowContentAccess = true // 웹뷰를 통해 content url에 접근할지 여부 + layoutAlgorithm = WebSettings.LayoutAlgorithm.TEXT_AUTOSIZING // 웹페이지의 레이아웃을 화면에 맞게 자동으로 조정 + } + } catch (ignore: NoSuchMethodError) {} + loadUrl(url) // 웹페이지 연결 + } + } fun chooseAdpater () { if (binding.missedCalls.isChecked) { @@ -482,69 +509,218 @@ internal class LauncherHome : Fragment() { } } + var maxDate : Long = Long.MIN_VALUE + var minDate : Long = Long.MAX_VALUE + val simpldateFormat = SimpleDateFormat("d MMM, yy",Locale.US) + fun jGuruMain(doc:Document) { + BLog.LOGE("do default parsing") + BLog.LOGE("SimpleDateFormat D MM yy => ${SimpleDateFormat("d MMM yy",Locale.US).format(Date())}") + val prevUrl = doc.getElementsByClass("prev").get(0).getElementsByAttribute("href").get(0).attr("href") + BLog.LOGE("doc.getElementsByClass(prev).get(0).html() ${prevUrl}") + doc.getElementsByClass("column").forEach { + var title = it.getElementsByAttribute("title").get(0).text() + var model = title.replace("[","").split("]")[0] + var pageLink = it.getElementsByClass("imgg").get(0).getElementsByAttribute("href").get(0).attr("href") + var imgg = it.getElementsByClass("imgg").get(0).getElementsByAttribute("src").get(0).attr("src") + var tags = it.getElementsByClass("tags").get(0).text() + var date = it.getElementsByClass("date").get(0).text() + var regDate = simpldateFormat.parse(date).time + + minDate = Math.min(minDate,regDate) + + maxDate = Math.max(maxDate,regDate) + + listItem.add(RssItem(model = model, title = title, pageLink = pageLink, image = imgg, tags = tags, date = simpldateFormat.parse(date).time)) + }.apply { + + BLog.LOGE("listItem.size >>> ${listItem.size}") + if (prevUrl!=null && prevUrl.length > TEST_PAG.length && prevUrl.contains("/page/") && (maxDate - minDate) < (1000L * 60L * 60L * 24L * 3L)) { + BLog.LOGE("listItem.size >>> ${listItem.size} do next ") + BLog.LOGE("saving data :: ${listItem.size}items ${simpldateFormat.format(Date(minDate))} ~ ${simpldateFormat.format(Date(maxDate))}") + binding.searcher01.postDelayed({binding.searcher01.loadUrl(prevUrl)}, 5000L) + } else { + listItem.sortByDescending { it.date } + BLog.LOGE("Stored data :: ${listItem.size}items ${simpldateFormat.format(Date(minDate))} ~ ${simpldateFormat.format(Date(maxDate))}") + Toast.makeText(requireContext(), + "Stored data :: ${listItem.size} items :: [${simpldateFormat.format(Date(minDate))} ~ ${simpldateFormat.format(Date(maxDate))}]", Toast.LENGTH_LONG).show() + } + } + } + + fun jGuruTag(doc: Document) { + listTags.clear() + doc.getElementsByTag("ul").forEach { + it.children().forEach { + if (it.tag().name.contains("li")) { + listTags.add( + RssTagItem( + tagTitle = it.getElementsByTag("a").get(0).text(), + link = it.getElementsByTag("a").get(0).attr("href") + ) + ) + + } + } + }.apply { + listTags.sortByDescending { it.count } + Toast.makeText(requireContext(), + "Stored data :: ${listTags.size}tags", Toast.LENGTH_SHORT).show() + } + } inner class MyJavaScriptInterface(val webView: WebView) { + @JavascriptInterface fun sendValueFromHtml(result: String) { - var htmlString = result.replace("\\u003","<") - val simpldateFormat = SimpleDateFormat("d MMM, yy",Locale.US) - val doc: Document = Jsoup.parse(htmlString) -// BLog.LOGE("binding.otherCheck in ThreadRun ${doc.body()}") - if(lastedFinishedPageUrl?.contains("page") == true || lastedFinishedPageUrl?.equals(TEST_PAG) == true) { - BLog.LOGE("do default parsing") - BLog.LOGE("SimpleDateFormat D MM yy => ${SimpleDateFormat("d MMM yy",Locale.US).format(Date())}") - val prevUrl = doc.getElementsByClass("prev").get(0).getElementsByAttribute("href").get(0).attr("href") - BLog.LOGE("doc.getElementsByClass(prev).get(0).html() ${prevUrl}") - doc.getElementsByClass("column").forEach { - var title = it.getElementsByAttribute("title").get(0).text() - var model = title.replace("[","").split("]")[0] - var pageLink = it.getElementsByClass("imgg").get(0).getElementsByAttribute("href").get(0).attr("href") - var imgg = it.getElementsByClass("imgg").get(0).getElementsByAttribute("src").get(0).attr("src") - var tags = it.getElementsByClass("tags").get(0).text() - var date = it.getElementsByClass("date").get(0).text() - listItem.add(RssItem(model = model, title = title, pageLink = pageLink, image = imgg, tags = tags, date = simpldateFormat.parse(date).time)) - }.apply { - BLog.LOGE("listItem.size >>> ${listItem.size}") - if (prevUrl!=null && prevUrl.length > TEST_PAG.length && prevUrl.contains("/page/") && listItem.size < (24 * 2) ) { - BLog.LOGE("listItem.size >>> ${listItem.size} do next ") - webView.postDelayed({webView.loadUrl(prevUrl)}, 5000L) - } else { - listItem.sortByDescending { it.date } - binding.mainList?.post { - binding.mainList.adapter = RssItemAdapter(listItem,requireContext()).apply { - updateData(listItem) + + if (lastedFinishedPageUrl.contains(TEST_PAG)) { + var htmlString = result.replace("\\u003","<") + val doc: Document = Jsoup.parse(htmlString) + if (lastedFinishedPageUrl?.contains("page") == true || lastedFinishedPageUrl?.equals( + TEST_PAG + ) == true + ) { + jGuruMain(doc) + } else if (lastedFinishedPageUrl?.contains("/tags/") == true) { + jGuruTag(doc) + } + } else if (lastedFinishedPageUrl?.contains("youtube") == true) { +// BLog.LOGE("doc youtube >>>> ${result}") + val doc: Document = Jsoup.parse(result) +// BLog.LOGE("doc.html().contains(ytInitialData) ${doc.html().contains("ytInitialData", false)}\n${doc.html().toString().split("var ytInitialData")[1].split("")[0]}") + + doc.getElementsByTag("script").forEach { + if(it.html().contains("var ytInitialData", false)) {/**/ + var ytInitialData = it.html().split("var ytInitialData = '")[1].split("'")[0].toString() + ytInitialData = ytInitialData.replace("\\x22", "\"") + .replace("\\x7b", "{") + .replace("\\x7d", "}") + .replace("\\x5b", "[") + .replace("\\x5d", "]") +// BLog.LOGE("doc youtube div >>>> ${StringEscapeUtils.escapeJson(ytInitialData)}") +// BLog.LOGE("doc youtube div >>>> ${StringEscapeUtils.escapeXml10(ytInitialData)}") +// BLog.LOGE("doc youtube div >>>> ${StringEscapeUtils.escapeJson(ytInitialData)}") +// BLog.LOGE("doc youtube div >>>> ${StringEscapeUtils.escapeHtml3(ytInitialData)}") +// BLog.LOGE("doc youtube div >>>> ${StringEscapeUtils.unescapeEcmaScript(ytInitialData)}") + var tempJSONObject : JSONObject? = null + JSONObject(StringEscapeUtils.unescapeEcmaScript(ytInitialData)).apply{ + tempJSONObject = this + }.keys().forEach { + if (it.equals("contents")) { + var child = tempJSONObject?.getJSONObject(it) + if(child?.length() == 1 && child?.has("singleColumnBrowseResultsRenderer") == true) { + BLog.LOGE("(singleColumnBrowseResultsRenderer >> ${child}") + var singleColumnBrowseResultsRenderer = child?.getJSONObject("singleColumnBrowseResultsRenderer") + if (singleColumnBrowseResultsRenderer?.has("tabs") == true) { + var tabs : JSONArray? = singleColumnBrowseResultsRenderer?.getJSONArray("tabs") + BLog.LOGE("(tabs >>>> ${tabs}") + for ( i in 0..<(tabs?.length() ?: 0)) { + var tabsChild = tabs?.getJSONObject(i) + BLog.LOGE("tabsChild >> ${tabsChild}") + if(tabsChild?.has("tabRenderer") == true) { + var tabRenderer = tabsChild?.getJSONObject("tabRenderer") + BLog.LOGE("tabRenderer >> ${tabRenderer}") + tabRenderer?.keys()?.forEach { + // BLog.LOGE("tabRenderer in key >> ${it}") + if (tabRenderer?.get(it) is String) { + BLog.LOGE("tabRenderer String in $it >> ${tabRenderer?.get(it)}") + } + else if (tabRenderer?.get(it) is JSONObject) { + var obj = tabRenderer?.getJSONObject(it) + BLog.LOGE("tabRenderer child JSONObject in $it >> ${obj}") + var pkey = it + obj?.keys()?.forEach { + jsonObjLog(pkey,it,obj) + } + } + else if (tabRenderer?.get(it) is JSONArray) { + BLog.LOGE("tabRenderer JSONArray in $it >> ${tabRenderer?.get(it)}") + } else { + BLog.LOGE("tabRenderer else in $it >> ${tabRenderer?.get(it)}") + } + } + } + } + } + } else { + child?.keys()?.forEach { childKey -> + BLog.LOGE("ytInitialData >>> ${it} >>>> ${childKey}") + } + } } } } } - } else if(lastedFinishedPageUrl?.contains("/tags/") == true ){ - listTags.clear() - doc.getElementsByTag("ul").forEach { - it.children().forEach { - if (it.tag().name.contains("li")) { - listTags.add( - RssTagItem( - tagTitle = it.getElementsByTag("a").get(0).text(), - link = it.getElementsByTag("a").get(0).attr("href") - ) - ) - - } - } - }.apply { - listTags.sortByDescending { it.count } - binding.mainList?.post { - binding.mainList.adapter = RssTagAdapter(listTags, requireContext()).apply { - updateData(listTags) - } - } - } +// ytChannel(doc.html()) } BLog.LOGE("binding.otherCheck after ThreadRun") } } + + fun jsonObjLog(pkey : String ,key : String, jsonObject: JSONObject) { + if (jsonObject?.has(key) == true && jsonObject?.get(key) is String) { + BLog.LOGE("jsonObjLog $pkey String in $key >> ${jsonObject?.getString(key)}") + } + else if (jsonObject?.has(key) == true && jsonObject?.get(key) is JSONObject) { + var obj = jsonObject?.getJSONObject(key) + BLog.LOGE("jsonObjLog $pkey JSONObject in $key >> ${obj}") + obj?.keys()?.forEach { + jsonObjLog(key,it, obj) + } + } + else if (jsonObject?.has(key) == true && jsonObject?.get(key) is JSONArray) { + BLog.LOGE("jsonObjLog $pkey JSONArray in $key >> ${jsonObject?.getJSONArray(key)}") + var array = jsonObject?.getJSONArray(key) + for ( i in 0..<(array?.length() ?: 0)) { + var child = array?.getJSONObject(i) + child?.keys()?.forEach { + jsonObjLog(key,it,child) + } + } + } else { + BLog.LOGE("jsonObjLog $pkey else in $key >> ${if(jsonObject?.has(key) == true) (jsonObject?.get(key)) else ("")}") + } + } + + fun ytChannel(dom: String) { + try { + + val parserFactory = XmlPullParserFactory.newInstance() + val kXmlParser = parserFactory.newPullParser() + val stream: InputStream = + ByteArrayInputStream(dom.toByteArray(Charset.forName("utf-8"))) + kXmlParser.setInput(stream, "utf-8") + var eventType = kXmlParser.eventType + var done = false + +// eventType이 End_Document가 아니고 done이 false일 경우 반복 + while(eventType != XmlPullParser.END_DOCUMENT) { + // 이름을 저장하기 위한 변수 + var name: String? = null + // eventType이 START_TAG인 경우 + if(eventType == XmlPullParser.START_TAG) { + // getName() 메서드를 사용해 현재 요소(태그)의 이름을 반환 + name = kXmlParser.name + BLog.LOGE("kXmlParser.name >> ${kXmlParser.name}") + BLog.LOGE("kXmlParser.name >> ${kXmlParser.text}") + BLog.LOGE("kXmlParser.name >> ${kXmlParser.attributeCount}") +// if(name == "city") { +// // getAttributeValue를 사용해 첫 번째 매개변수 및 +// // 두 번째 매개변수로 식별된 특성 값 획득 +// cityView.text = kXmlParser.getAttributeValue(null, "name") +// } else if(name == "temperature") { +// temperatureView.text = kXmlParser.getAttributeValue(null, "value") +// } + } + // 다음 이벤트로 넘기기 + eventType = kXmlParser.next() + } + } catch(e: Exception) { + + } + } /* gestures on root view */ @SuppressLint("ClickableViewAccessibility") private fun rootViewGestures() { diff --git a/app/src/main/kotlin/rasel/lunar/launcher/utils/RssList.kt b/app/src/main/kotlin/rasel/lunar/launcher/utils/RssList.kt new file mode 100644 index 00000000..6a790814 --- /dev/null +++ b/app/src/main/kotlin/rasel/lunar/launcher/utils/RssList.kt @@ -0,0 +1,31 @@ +package rasel.lunar.launcher.utils + +import android.widget.Toast +import org.jsoup.nodes.Document +import rasel.lunar.launcher.LauncherActivity.Companion.TEST_PAG +import rasel.lunar.launcher.home.LauncherHome.Companion.listItem +import rasel.lunar.launcher.home.RssItem +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale + +object RssList { + val youtubeUrls = arrayListOf( + "https://www.youtube.com/@zzanbro", + "https://www.youtube.com/@sungsikyung", + "https://www.youtube.com/@%EC%A7%80%EB%AC%B4%EB%B9%84", + "https://www.youtube.com/@gyeomsonisnothing", + "https://www.youtube.com/@ddeunddeun" + ) +} + +object DocParserManager { + fun parse(doc : Document) {} +} +interface DocParser { fun parse(doc : Document) : T } + +class JGuruMain { + var maxDate : Long = Long.MIN_VALUE + var minDate : Long = Long.MAX_VALUE + +} \ No newline at end of file diff --git a/app/src/main/kotlin/rasel/lunar/launcher/utils/StringEscapeUtils.kt b/app/src/main/kotlin/rasel/lunar/launcher/utils/StringEscapeUtils.kt new file mode 100644 index 00000000..e69de29b