karolina/weserplaner
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PARSING HTML WORKS AND IT GETS SAVED TO THE DATABASE WOOO

Browse Source
This commit is contained in:
denizk0461
2023-04-10 20:54:15 +02:00
parent 97bc2b1068
commit cab50b5f2d
13 changed files with 161 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/denizk0461/studip/adapter/StudIPEventAdapter.kt
+1 -1
View File
@@ -26,7 +26,7 @@ class StudIPEventAdapter(private val events: List<StudIPEvent>) : RecyclerView.A
holder.binding.textTitle.text = currentItem.title
holder.binding.textLecturers.text = currentItem.lecturer
holder.binding.textRoom.text = currentItem.room
holder.binding.textTimeslot.text = currentItem.timeslot
holder.binding.textTimeslot.text = currentItem.timeslot()
// TODO inflate view saying "no events!" or sth
app/src/main/java/com/denizk0461/studip/data/DummyData.kt
+41
View File
@@ -13,4 +13,45 @@ object DummyData {
StudIPEvent(6, "Gesellschaft und Raum", "Lossau, Mossig", "GW2 B1820", 3, 1, 1),
StudIPEvent(7, "Gesellschaft und Raum2", "Lossau, Mossig", "GW2 B1820", 4, 4, 2),
)
val timeslotStartMap: Map<String, Int> = mapOf(
"6:00" to 0,
"6:15" to 0,
"8:00" to 1,
"8:15" to 1,
"10:00" to 2,
"10:15" to 2,
"12:00" to 3,
"12:15" to 3,
"14:00" to 4,
"14:15" to 4,
"16:00" to 5,
"16:15" to 5,
"18:00" to 6,
"18:15" to 6,
)
val timeslotEndMap: Map<String, Int> = mapOf(
"8:00" to 0,
"7:45" to 0,
"10:00" to 1,
"9:45" to 1,
"12:00" to 2,
"11:45" to 2,
"14:00" to 3,
"13:45" to 3,
"16:00" to 4,
"15:45" to 4,
"18:00" to 5,
"17:45" to 5,
"20:00" to 6,
"19:45" to 6,
)
fun parseTimeslot(time: String): Pair<Int, Int> {
val splitTime = time.split(" - ")
val start = timeslotStartMap[splitTime[0]] ?: 0
val length = (timeslotEndMap[splitTime[1]] ?: 0)
return Pair(start, length)
}
}
app/src/main/java/com/denizk0461/studip/data/StudIPParser.kt
+49
View File
@@ -0,0 +1,49 @@
package com.denizk0461.studip.data
import android.util.Log
import com.denizk0461.studip.model.StudIPEvent
import org.jsoup.Jsoup
class StudIPParser {
fun parse(html: String, insert: (events: List<StudIPEvent>) -> Unit) {
var id = 0
val doc = Jsoup.parse(html)
Log.d("HELLO2", "doc: $doc")
val newEvents = mutableListOf<StudIPEvent>()
val columns = arrayOf(
doc.getElementById("calendar_view_1_column_0"),
doc.getElementById("calendar_view_1_column_1"),
doc.getElementById("calendar_view_1_column_2"),
doc.getElementById("calendar_view_1_column_3"),
doc.getElementById("calendar_view_1_column_4"),
)
Log.d("HELLO2", "days found: ${columns.size}, element 1: ${columns[0]}")
columns.forEachIndexed { index, element ->
element?.getElementsByClass("schedule_entry")?.forEachIndexed { entryIndex, entry ->
Log.d("HELLO2", "element - $entryIndex - found: $entry")
val entryHeader = entry.getElementsByTag("dt")[0].text()
val delimiter = entryHeader.lastIndexOf('(')
val parsedTitle = entryHeader.substring(0 until delimiter)
val parsedLecturers = entryHeader.substring(delimiter + 1 until entryHeader.length - 1)
val entryInfos = entry.getElementsByTag("dd")[0].text().split(", ")
val timeSlot = DummyData.parseTimeslot(entryInfos[0])
val event = StudIPEvent(
id = id,
title = parsedTitle,
lecturer = parsedLecturers,
room = entryInfos[1],
day = index,
timeslotStart = timeSlot.first,
timeslotEnd = timeSlot.second,
)
newEvents.add(event)
id += 1
Log.d("HELLO2", event.toString())
}
}
insert(newEvents)
}
}
app/src/main/java/com/denizk0461/studip/data/StudIPScraper.kt
-29
View File
@@ -1,29 +0,0 @@
package com.denizk0461.studip.data
import android.content.Context
import android.graphics.Bitmap
import android.webkit.WebView
import android.webkit.WebViewClient
class StudIPScraper : WebViewClient() {
fun parse(context: Context) {
// val res = Jsoup.connect("https://elearning.uni-bremen.de/index.php?again=yes")
// .data("loginname", "deniz7", "password", "xSh4d0wZ3r0_+nbrmn")
// .method(Connection.Method.POST)
// .execute()
// val doc = Jsoup.connect("https://elearning.uni-bremen.de/dispatch.php/calendar/schedule").get()
// Log.d("HELLO", doc.toString())
}
override fun onPageStarted(view: WebView?, url: String?, favicon: Bitmap?) {
super.onPageStarted(view, url, favicon)
}
override fun onPageFinished(view: WebView?, url: String?) {
super.onPageFinished(view, url)
}
}
app/src/main/java/com/denizk0461/studip/db/EventDAO.kt
+3
View File
@@ -15,6 +15,9 @@ interface EventDAO {
@Insert
fun insertEvent(event: StudIPEvent)
@Insert
fun insertEvents(event: List<StudIPEvent>)
@Query("DELETE FROM events")
fun nukeEvents()
}
app/src/main/java/com/denizk0461/studip/db/EventDatabase.kt
+2 -2
View File
@@ -6,7 +6,7 @@ import androidx.room.Room
import androidx.room.RoomDatabase
import com.denizk0461.studip.model.StudIPEvent
@Database(entities = [StudIPEvent::class], version = 1)
@Database(entities = [StudIPEvent::class], version = 2)
abstract class EventDatabase : RoomDatabase() {
abstract fun dao(): EventDAO
@@ -21,7 +21,7 @@ abstract class EventDatabase : RoomDatabase() {
context.applicationContext,
EventDatabase::class.java,
"event_db"
).build()
).fallbackToDestructiveMigration().build()
}
}
return instance!!
app/src/main/java/com/denizk0461/studip/db/EventRepository.kt
+1
View File
@@ -12,4 +12,5 @@ class EventRepository(app: Application) {
fun nukeEvents() { dao.nukeEvents() }
fun insertEvent(event: StudIPEvent) { dao.insertEvent(event) }
fun insertEvents(events: List<StudIPEvent>) { dao.insertEvents(events) }
}
app/src/main/java/com/denizk0461/studip/fragment/EventFragment.kt
+4 -3
View File
@@ -12,7 +12,7 @@ import androidx.recyclerview.widget.PagerSnapHelper
import com.denizk0461.studip.R
import com.denizk0461.studip.adapter.StudIPEventPageAdapter
import com.denizk0461.studip.data.DummyData
import com.denizk0461.studip.data.StudIPScraper
import com.denizk0461.studip.data.StudIPParser
import com.denizk0461.studip.databinding.FragmentEventBinding
import com.denizk0461.studip.viewmodel.EventViewModel
@@ -40,16 +40,17 @@ class EventFragment : Fragment() {
super.onViewCreated(view, savedInstanceState)
viewModel.allEvents.observe(viewLifecycleOwner) { events ->
recyclerViewAdapter = StudIPEventPageAdapter(DummyData.events.toList())
recyclerViewAdapter = StudIPEventPageAdapter(events)//DummyData.events.toList())
binding.recyclerView.adapter = recyclerViewAdapter
recyclerViewLayoutManager =
LinearLayoutManager(context, LinearLayoutManager.HORIZONTAL, false)
binding.recyclerView.layoutManager = recyclerViewLayoutManager
binding.recyclerView.onFlingListener = null
PagerSnapHelper().attachToRecyclerView(binding.recyclerView)
binding.recyclerView.scheduleLayoutAnimation()
}
viewModel.doAsync { StudIPScraper().parse(requireContext()) }
// viewModel.doAsync { StudIPParser().parse(requireContext()) }
binding.fab.setOnClickListener { view ->
launchWebview()
app/src/main/java/com/denizk0461/studip/fragment/SecondFragment.kt → app/src/main/java/com/denizk0461/studip/fragment/ParserFragment.kt
+28 -12
View File
@@ -1,31 +1,33 @@
package com.denizk0461.studip.fragment
import android.os.Bundle
import android.util.Log
import androidx.fragment.app.Fragment
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import android.webkit.ValueCallback
import android.webkit.WebChromeClient
import android.webkit.WebResourceRequest
import android.webkit.WebView
import android.webkit.WebViewClient
import androidx.navigation.fragment.findNavController
import com.denizk0461.studip.R
import androidx.fragment.app.Fragment
import androidx.fragment.app.viewModels
import com.denizk0461.studip.data.StudIPParser
import com.denizk0461.studip.databinding.FragmentSecondBinding
import com.denizk0461.studip.viewmodel.ParserViewModel
import java.net.URLDecoder
/**
* A simple [Fragment] subclass as the second destination in the navigation.
*/
class SecondFragment : Fragment() {
class ParserFragment : Fragment() {
private var _binding: FragmentSecondBinding? = null
private var html = "" // temporary storage for the website HTML
// This property is only valid between onCreateView and
// onDestroyView.
private val binding get() = _binding!!
private val viewModel: ParserViewModel by viewModels()
override fun onCreateView(
inflater: LayoutInflater, container: ViewGroup?,
savedInstanceState: Bundle?
@@ -43,24 +45,38 @@ class SecondFragment : Fragment() {
// findNavController().navigate(R.id.action_SecondFragment_to_FirstFragment)
// }
binding.webview.settings.javaScriptEnabled = true
binding.webview.settings.apply {
javaScriptEnabled = true
}
binding.webview.webViewClient = object : WebViewClient() {
override fun shouldOverrideUrlLoading(
view: WebView?,
request: WebResourceRequest?
view: WebView,
request: WebResourceRequest
): Boolean {
return false
}
override fun onPageFinished(view: WebView, url: String) {
binding.webview.loadUrl(
"javascript:window.HtmlViewer.showHTML" +
"('<html>'+document.getElementsByTagName('html')[0].innerHTML+'</html>');"
)
}
}
binding.webview.loadUrl("https://elearning.uni-bremen.de/index.php?again=yes")
binding.fab.setOnClickListener { view ->
binding.webview.evaluateJavascript(
"(function(){return window.document.body.outerHTML})();"
"(function(){return encodeURI(document.getElementsByTagName('html')[0].innerHTML)})();"
) { p0 ->
Log.d("HELLO", p0 ?: "nothing returned")
viewModel.nukeEvents()
StudIPParser().parse(URLDecoder.decode(p0, "UTF-8")) { events ->
viewModel.insertEvents(events)
}
}
// html.chunked(3000).forEach {
// Log.d("HELLO", it)
// }
}
}
app/src/main/java/com/denizk0461/studip/model/StudIPEvent.kt
+19 -9
View File
@@ -1,5 +1,6 @@
package com.denizk0461.studip.model
import android.util.Log
import androidx.room.Entity
import androidx.room.Ignore
import androidx.room.PrimaryKey
@@ -11,15 +12,26 @@ data class StudIPEvent(
val lecturer: String, // the lecturer(s)
val room: String, // the room the event takes place in
val day: Int, // 0 = Monday, 1 = Tuesday, 2 = Wednesday, 3 = Thursday, 4 = Friday
val timeslotStart: Int, // 0 = 08:15, 1 = 10:15, 2 = 12:15, 3 = 14:15, 4 = 16:15, 5 = 18:15
val timeslots: Int, // the amount of timeslots the course takes place over (1 timeslot = 90min)
val timeslotStart: Int, // 1 = 08:15, 2 = 10:15, 3 = 12:15, 4 = 14:15, 5 = 16:15, 6 = 18:15
val timeslotEnd: Int, // 1 = 09:45 etc.
) {
@Ignore private val timeSlotStartTimes: List<String> = listOf("08:15", "10:15", "12:15", "14:15", "16:15", "18:15")
@Ignore private val timeSlotEndTimes: List<String> = listOf("09:45", "11:45", "13:45", "15:45", "17:45", "19:45")
// constructor(
// val title: String,
// val lecturer: String,
// val room: String,
// val day: Int,
// val
// ) : this(title, lecturer, room, day, 1, 1)
@Ignore val timeslot: String =
"${timeSlotStartTimes[timeslotStart]} – ${timeSlotEndTimes[timeslotStart + timeslots - 1]}"
@Ignore private val timeSlotStartTimes: List<String> = listOf("06:15", "08:15", "10:15", "12:15", "14:15", "16:15", "18:15")
@Ignore private val timeSlotEndTimes: List<String> = listOf("07:45", "09:45", "11:45", "13:45", "15:45", "17:45", "19:45")
fun timeslot(): String {
Log.d("HELLO3", title)
Log.d("HELLO3", "s $timeslotStart st $timeslotEnd")
return "${timeSlotStartTimes[timeslotStart]} – ${timeSlotEndTimes[timeslotEnd]}"
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
@@ -32,7 +44,6 @@ data class StudIPEvent(
return true
}
override fun hashCode(): Int {
var result = id
result = 31 * result + title.hashCode()
@@ -40,10 +51,9 @@ data class StudIPEvent(
result = 31 * result + room.hashCode()
result = 31 * result + day
result = 31 * result + timeslotStart
result = 31 * result + timeslots
result = 31 * result + timeslotEnd
result = 31 * result + timeSlotStartTimes.hashCode()
result = 31 * result + timeSlotEndTimes.hashCode()
result = 31 * result + timeslot.hashCode()
return result
}
}
app/src/main/java/com/denizk0461/studip/viewmodel/ParserViewModel.kt
+11
View File
@@ -0,0 +1,11 @@
package com.denizk0461.studip.viewmodel
import android.app.Application
import com.denizk0461.studip.model.StudIPEvent
class ParserViewModel(app: Application) : TemplateViewModel(app) {
fun insertEvent(event: StudIPEvent) { repo.insertEvent(event) }
fun insertEvents(events: List<StudIPEvent>) { doAsync { repo.insertEvents(events) } }
fun nukeEvents() { doAsync { repo.nukeEvents() } }
}
app/src/main/res/layout/fragment_second.xml
+1 -1
View File
@@ -4,7 +4,7 @@
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
tools:context=".fragment.SecondFragment">
tools:context=".fragment.ParserFragment">
<WebView
android:id="@+id/webview"
app/src/main/res/navigation/nav_graph.xml
+1 -1
View File
@@ -17,7 +17,7 @@
</fragment>
<fragment
android:id="@+id/SecondFragment"
android:name="com.denizk0461.studip.fragment.SecondFragment"
android:name="com.denizk0461.studip.fragment.ParserFragment"
android:label="@string/second_fragment_label"
tools:layout="@layout/fragment_second">