refactor: Task 18 완료 - BaseGetter Jsoup 스크래핑 템플릿 도입 및 ClienGetter 적용
- BaseGetter.kt: * fetchDocument(url, timeoutMs): User-Agent, 타임아웃 설정 및 URL 단위 예외 격리를 포함한 공통 Jsoup Document 로더 추가 * scrapeUrls(urls, timeoutMs, onDocument): temp 초기화 및 URL별 독립 파싱/예외 격리 템플릿 메서드 추가 - ClienGetter.kt: realWork() 내 반복 Jsoup.connect 보일러플레이트를 scrapeUrls() 호출로 단순화 - TODO.md: 4-5 항목 완료 처리 - 검증: ./gradlew :app:compileDebugKotlin 빌드 성공 확인 (2s)
This commit is contained in:
@@ -34,7 +34,35 @@ abstract class BaseGetter(internal val context: Context) {
|
||||
|
||||
abstract fun realWork() : List<RealmObject>
|
||||
open suspend fun fetchData(): List<RealmObject> {
|
||||
|
||||
return realWork()
|
||||
}
|
||||
|
||||
protected fun fetchDocument(url: String, timeoutMs: Int = 15_000): org.jsoup.nodes.Document? {
|
||||
return try {
|
||||
org.jsoup.Jsoup.connect(url)
|
||||
.userAgent(USAGT)
|
||||
.timeout(timeoutMs)
|
||||
.get()
|
||||
} catch (e: Exception) {
|
||||
bums.lunatic.launcher.utils.Blog.LOGE("${this::class.simpleName} fetchDocument failed ($url): ${e.message}")
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
protected inline fun scrapeUrls(
|
||||
urls: Iterable<String>,
|
||||
timeoutMs: Int = 15_000,
|
||||
onDocument: (url: String, doc: org.jsoup.nodes.Document) -> Unit
|
||||
) {
|
||||
temp.clear()
|
||||
urls.forEach { url ->
|
||||
fetchDocument(url, timeoutMs)?.let { doc ->
|
||||
try {
|
||||
onDocument(url, doc)
|
||||
} catch (e: Exception) {
|
||||
bums.lunatic.launcher.utils.Blog.LOGE("${this::class.simpleName} parse failed ($url): ${e.message}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,21 +52,17 @@ class ClienGetter(context: Context) : BaseGetter(context = context) {
|
||||
override fun realWork(): List<RealmObject> {
|
||||
RssDataType.CLIEN.isOn {
|
||||
Blog.LOGE("realWork() ${this::class.simpleName}")
|
||||
try {
|
||||
temp.clear()
|
||||
val testUrl2 = arrayListOf("https://www.clien.net/service/group/community","https://www.clien.net/service/board/park","https://www.clien.net/service/board/news","https://www.clien.net/service/board/useful","https://www.clien.net/service/board/pds")
|
||||
testUrl2.forEach { url ->
|
||||
Jsoup.connect(url)
|
||||
.userAgent(USAGT)
|
||||
.get().let { ruli ->
|
||||
// BLog.LOGE("test ${url} >> ${ruli.title()}")
|
||||
ruli.getElementsByClass("list_item symph_row ").forEach { ruli_tr ->
|
||||
parseClien(ruli_tr)
|
||||
}
|
||||
}
|
||||
val urls = listOf(
|
||||
"https://www.clien.net/service/group/community",
|
||||
"https://www.clien.net/service/board/park",
|
||||
"https://www.clien.net/service/board/news",
|
||||
"https://www.clien.net/service/board/useful",
|
||||
"https://www.clien.net/service/board/pds"
|
||||
)
|
||||
scrapeUrls(urls) { _, doc ->
|
||||
doc.getElementsByClass("list_item symph_row ").forEach { row ->
|
||||
parseClien(row)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
return temp
|
||||
|
||||
Reference in New Issue
Block a user