karolina/weserplaner
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented canteen scraper/parser functionality

Browse Source
This commit is contained in:
denizk0461
2023-04-16 18:19:06 +02:00
parent 2117535f63
commit 5543eb4e32
15 changed files with 270 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/java/com/denizk0461/studip/activity/MainActivity.kt
+2 -2
View File
@@ -13,7 +13,7 @@ import com.denizk0461.studip.data.Dependencies
import com.denizk0461.studip.databinding.ActivityMainBinding
import com.denizk0461.studip.db.EventRepository
import com.denizk0461.studip.fragment.EventFragment
import com.denizk0461.studip.fragment.FoodFragment
import com.denizk0461.studip.fragment.CanteenFragment
import com.denizk0461.studip.fragment.SettingsFragment
class MainActivity : AppCompatActivity() {
@@ -21,7 +21,7 @@ class MainActivity : AppCompatActivity() {
private lateinit var appBarConfiguration: AppBarConfiguration
private lateinit var binding: ActivityMainBinding
private var currentFragment = FragmentType.SCHEDULE
private val fragments = listOf(EventFragment(), FoodFragment(), SettingsFragment())
private val fragments = listOf(EventFragment(), CanteenFragment(), SettingsFragment())
override fun onCreate(savedInstanceState: Bundle?) {
WindowCompat.setDecorFitsSystemWindows(window, false)
app/src/main/java/com/denizk0461/studip/adapter/StudIPEventAdapter.kt
+2 -2
View File
@@ -48,11 +48,11 @@ class StudIPEventAdapter(
if (!isAnyCourseHighlighted && currentItem.isCurrentCourse(currentCalendar)) { // highlight
isAnyCourseHighlighted = true
holder.binding.cardBackground.strokeColor = colorPrimary.data
holder.binding.cardBackground.strokeWidth = 6
holder.binding.cardBackground.strokeWidth = 6 // TODO replace this with proper float -> dp conversion
// R.color.list_card_selected)
} else { // un-highlight
holder.binding.cardBackground.strokeColor = colorTextHint.data
holder.binding.cardBackground.strokeWidth = 3
holder.binding.cardBackground.strokeWidth = 3 // this should (?) be 1dp
}
holder.binding.cardBackground.setOnClickListener {
app/src/main/java/com/denizk0461/studip/data/StwParser.kt
+144
View File
@@ -0,0 +1,144 @@
package com.denizk0461.studip.data
import android.util.Log
import com.denizk0461.studip.model.CanteenOffer
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
// Parser class for the Studierendenwerk cafeteria plans
class StwParser {
private var id = 0
fun parse(onFinish: () -> Unit) {
id = 0
val links = mutableListOf<String>()
if (true) { links.add(urlUniMensa) }
// if (true) { links.add(urlCafeCentral) }
// if (true) { links.add(urlNW1) }
// if (true) { links.add(urlGW2) }
// if (true) { links.add(urlHSBNeustadt) }
// if (true) { links.add(urlHSBAirport) }
// if (true) { links.add(urlHSBWerder) }
// if (true) { links.add(urlHfK) }
// if (true) { links.add(urlMensaBHV) }
// if (true) { links.add(urlCafeBHV) }
Dependencies.repo.nukeOffers()
links.forEach { link ->
Dependencies.repo.insertOffers(parseFromPage(link))
}
}
private fun parseFromPage(url: String): List<CanteenOffer> {
val items = mutableListOf<CanteenOffer>()
var date = ""
var day = 0
val doc = Jsoup.connect(url).get()
doc.getElementsByClass("food-plan").forEach { dayPlan ->
val rawDate = doc.getElementsByClass("tabs")[0]
.getElementsByClass("tab-date")[day].text().split(" ")
date = "${rawDate[0]}${rawDate[1].monthToNumber()}."
day += 1
dayPlan.getElementsByClass("food-category").forEach { category ->
val categoryTitle = category.getElementsByClass("category-name")[0].text()
category
.getElementsByTag("tbody")[0]
.getElementsByTag("tr").forEach { element ->
val tableRows = element.getElementsByTag("td")
val prefs =
element.getElementsByClass("field field-name-field-food-types")[0]
val newItem = CanteenOffer(
id = id,
date = date,
category = categoryTitle,
title = tableRows[1].getFilteredText(),
price = tableRows.getTextOrEmpty(2),
isFair = prefs.isDietaryPreferenceMet(PREFERENCE_FAIR),
isFish = prefs.isDietaryPreferenceMet(PREFERENCE_FISH),
isPoultry = prefs.isDietaryPreferenceMet(PREFERENCE_POULTRY),
isLamb = prefs.isDietaryPreferenceMet(PREFERENCE_LAMB),
isVital = prefs.isDietaryPreferenceMet(PREFERENCE_VITAL),
isBeef = prefs.isDietaryPreferenceMet(PREFERENCE_BEEF),
isPork = prefs.isDietaryPreferenceMet(PREFERENCE_PORK),
isVegan = prefs.isDietaryPreferenceMet(PREFERENCE_VEGAN),
isVegetarian = prefs.isDietaryPreferenceMet(PREFERENCE_VEGETARIAN),
isGame = prefs.isDietaryPreferenceMet(PREFERENCE_GAME),
)
items.add(newItem)
id += 1
Log.d("eek!", newItem.toString())
}
}
}
return items
}
private fun Element.isDietaryPreferenceMet(preference: String): Boolean =
getElementsByAttributeValue("src", preference).isNotEmpty()
private fun Element.getFilteredText(): String {
var text = html()
while (text.contains("<sup>")) {
text = text.substring(0 until text.indexOf("<sup>")) + text.substring(text.indexOf("</sup>")+6 until text.length)
}
return text
}
private fun Elements.getTextOrEmpty(index: Int): String = try {
get(index).text()
} catch (e: java.lang.IndexOutOfBoundsException) {
""
}
private fun String.monthToNumber(): String = when (this) {
"Jan" -> "01"
"Feb" -> "02"
"Mar" -> "03"
"Apr" -> "04"
"Mai" -> "05"
"Jun" -> "06"
"Jul" -> "07"
"Aug" -> "08"
"Sep" -> "09"
"Okt" -> "10"
"Nov" -> "11"
"Dez" -> "12"
else -> "00" // shouldn't occur
}
private val PREFERENCE_FAIR = "https://www.stw-bremen.de/sites/default/files/images/pictograms/at_small.png"
private val PREFERENCE_FISH = "https://www.stw-bremen.de/sites/default/files/images/pictograms/fisch.png"
private val PREFERENCE_POULTRY = "https://www.stw-bremen.de/sites/default/files/images/pictograms/geflugel.png"
private val PREFERENCE_LAMB = "https://www.stw-bremen.de/sites/default/files/images/pictograms/lamm.png"
private val PREFERENCE_VITAL = "https://www.stw-bremen.de/sites/default/files/images/pictograms/mensa_vital.png"
private val PREFERENCE_BEEF = "https://www.stw-bremen.de/sites/default/files/images/pictograms/rindfleisch.png"
private val PREFERENCE_PORK = "https://www.stw-bremen.de/sites/default/files/images/pictograms/schwein.png"
private val PREFERENCE_VEGAN = "https://www.stw-bremen.de/sites/default/files/images/pictograms/mensa_vegan.png"
private val PREFERENCE_VEGETARIAN = "https://www.stw-bremen.de/sites/default/files/images/pictograms/vegetarisch.png"
private val PREFERENCE_GAME = "https://www.stw-bremen.de/sites/default/files/images/pictograms/wild.png"
private val urlUniMensa = "https://www.stw-bremen.de/de/mensa/uni-mensa"
private val urlCafeCentral = "https://www.stw-bremen.de/de/mensa/cafe-central"
private val urlNW1 = "https://www.stw-bremen.de/de/mensa/nw-1"
private val urlGW2 = "https://www.stw-bremen.de/de/cafeteria/gw2"
// private val urlGraz = ""
private val urlHSBNeustadt = "https://www.stw-bremen.de/de/mensa/neustadtswall"
private val urlHSBWerder = "https://www.stw-bremen.de/de/mensa/werderstra%C3%9Fe"
private val urlHSBAirport = "https://www.stw-bremen.de/de/mensa/airport"
private val urlHfK = "https://www.stw-bremen.de/de/mensa/interimsmensa-hfk"
private val urlMensaBHV = "https://www.stw-bremen.de/de/mensa/bremerhaven"
private val urlCafeBHV = "https://www.stw-bremen.de/de/cafeteria/bremerhaven"
}
app/src/main/java/com/denizk0461/studip/db/EventDAO.kt
+10
View File
@@ -4,6 +4,7 @@ import androidx.lifecycle.LiveData
import androidx.room.Dao
import androidx.room.Insert
import androidx.room.Query
import com.denizk0461.studip.model.CanteenOffer
import com.denizk0461.studip.model.StudIPEvent
@Dao
@@ -20,4 +21,13 @@ interface EventDAO {
@Query("DELETE FROM events")
fun nukeEvents()
@get:Query("SELECT * FROM offers ORDER BY id")
val allOffers: LiveData<List<CanteenOffer>>
@Insert
fun insertOffers(offers: List<CanteenOffer>)
@Query("DELETE from offers")
fun nukeOffers()
}
app/src/main/java/com/denizk0461/studip/db/EventDatabase.kt
+2 -1
View File
@@ -4,9 +4,10 @@ import android.content.Context
import androidx.room.Database
import androidx.room.Room
import androidx.room.RoomDatabase
import com.denizk0461.studip.model.CanteenOffer
import com.denizk0461.studip.model.StudIPEvent
@Database(entities = [StudIPEvent::class], version = 3)
@Database(entities = [StudIPEvent::class, CanteenOffer::class], version = 6)
abstract class EventDatabase : RoomDatabase() {
abstract fun dao(): EventDAO
app/src/main/java/com/denizk0461/studip/db/EventRepository.kt
+7 -1
View File
@@ -2,6 +2,7 @@ package com.denizk0461.studip.db
import android.app.Application
import androidx.lifecycle.LiveData
import com.denizk0461.studip.model.CanteenOffer
import com.denizk0461.studip.model.StudIPEvent
class EventRepository(app: Application) {
@@ -10,7 +11,12 @@ class EventRepository(app: Application) {
val allEvents: LiveData<List<StudIPEvent>> = dao.allEvents
fun nukeEvents() { dao.nukeEvents() }
fun insertEvent(event: StudIPEvent) { dao.insertEvent(event) }
fun insertEvents(events: List<StudIPEvent>) { dao.insertEvents(events) }
fun nukeEvents() { dao.nukeEvents() }
val allOffers: LiveData<List<CanteenOffer>> = dao.allOffers
fun insertOffers(offers: List<CanteenOffer>) { dao.insertOffers(offers) }
fun nukeOffers() { dao.nukeOffers() }
}
app/src/main/java/com/denizk0461/studip/fragment/CanteenFragment.kt
+46
View File
@@ -0,0 +1,46 @@
package com.denizk0461.studip.fragment
import android.os.Bundle
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import androidx.fragment.app.Fragment
import androidx.fragment.app.viewModels
import com.denizk0461.studip.databinding.FragmentCanteenBinding
import com.denizk0461.studip.viewmodel.CanteenViewModel
class CanteenFragment : Fragment() {
private var _binding: FragmentCanteenBinding? = null
// This property is only valid between onCreateView and
// onDestroyView.
private val binding get() = _binding!!
private val viewModel: CanteenViewModel by viewModels()
override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
_binding = FragmentCanteenBinding.inflate(inflater, container, false)
return binding.root
}
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
viewModel.allOffers.observe(viewLifecycleOwner) { offers ->
// TODO this must be changed. if an exception is raised, then this will not fire
binding.swipeRefreshLayout.isRefreshing = false
}
binding.swipeRefreshLayout.setOnRefreshListener {
viewModel.fetchOffers {
// binding.swipeRefreshLayout.isRefreshing = false
}
}
}
override fun onDestroyView() {
super.onDestroyView()
_binding = null
}
}
app/src/main/java/com/denizk0461/studip/fragment/EventFragment.kt
+1 -20
View File
@@ -1,15 +1,12 @@
package com.denizk0461.studip.fragment
import android.content.Intent
import android.os.Bundle
import androidx.fragment.app.Fragment
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import androidx.fragment.app.viewModels
import androidx.navigation.fragment.NavHostFragment.Companion.findNavController
import com.denizk0461.studip.R
import com.denizk0461.studip.activity.FetcherActivity
import com.denizk0461.studip.adapter.StudIPEventAdapter
import com.denizk0461.studip.adapter.StudIPEventPageAdapter
import com.denizk0461.studip.databinding.FragmentEventBinding
@@ -59,7 +56,7 @@ class EventFragment : Fragment() {
viewPagerAdapter = StudIPEventPageAdapter(listOf(), object : StudIPEventAdapter.OnClickListener {
override fun onClick(event: StudIPEvent) {
// do nothing
}
override fun onLongClick(event: StudIPEvent) {
@@ -76,10 +73,6 @@ class EventFragment : Fragment() {
switchToCurrentDayView()
}
binding.fab.setOnClickListener { view ->
launchWebview()
}
// binding.buttonFirst.setOnClickListener {
// findNavController().navigate(R.id.action_FirstFragment_to_SecondFragment)
// }
@@ -99,16 +92,4 @@ class EventFragment : Fragment() {
binding.viewPager.currentItem = dayOfWeek
// binding.dayTabLayout.getTabAt(dayOfWeek)?.select()
}
private fun launchWebview() {
// findNavController(this).navigate(R.id.action_FirstFragment_to_SecondFragment)
startActivity(Intent(context, FetcherActivity::class.java))
}
/**
* Highlights the current or next course.
*/
private fun highlightCurrentCourse() {
}
}
app/src/main/java/com/denizk0461/studip/fragment/FoodFragment.kt
-32
View File
@@ -1,32 +0,0 @@
package com.denizk0461.studip.fragment
import android.os.Bundle
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import androidx.fragment.app.Fragment
import com.denizk0461.studip.databinding.FragmentFoodBinding
class FoodFragment : Fragment() {
private var _binding: FragmentFoodBinding? = null
// This property is only valid between onCreateView and
// onDestroyView.
private val binding get() = _binding!!
override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
_binding = FragmentFoodBinding.inflate(inflater, container, false)
return binding.root
}
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
}
override fun onDestroyView() {
super.onDestroyView()
_binding = null
}
}
app/src/main/java/com/denizk0461/studip/fragment/SettingsFragment.kt
+12 -17
View File
@@ -1,8 +1,8 @@
package com.denizk0461.studip.fragment
import android.annotation.SuppressLint
import android.content.Intent
import android.net.Uri
import android.os.Build
import android.os.Bundle
import android.view.LayoutInflater
import android.view.View
@@ -10,6 +10,7 @@ import android.view.ViewGroup
import android.widget.Toast
import androidx.fragment.app.Fragment
import com.denizk0461.studip.BuildConfig
import com.denizk0461.studip.activity.FetcherActivity
import com.denizk0461.studip.data.Misc
import com.denizk0461.studip.databinding.FragmentSettingsBinding
@@ -31,15 +32,15 @@ class SettingsFragment : Fragment() {
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
var toast2 = makeToast("2")
var toast22 = makeToast("22")
var toast222 = makeToast("222")
// binding.layoutQuarter.setOnClickListener {
// isQuarterChecked = !isQuarterChecked
// binding.switchQuarter.isChecked = isQuarterChecked
// }
binding.buttonRefreshSchedule.setOnClickListener {
launchWebView()
}
binding.switchQuarter.setOnClickListener {
isQuarterChecked = !isQuarterChecked
}
@@ -47,24 +48,15 @@ class SettingsFragment : Fragment() {
binding.buttonAppVersion.setOnClickListener {
when (appVersionClick) {
19 -> {
toast2.show()
appVersionClick += 1
}
20 -> {
toast2.cancel()
toast22.show()
appVersionClick += 1
}
21 -> {
toast22.cancel()
toast222.show()
appVersionClick += 1
}
22 -> {
toast222.cancel()
toast2 = makeToast("2")
toast22 = makeToast("22")
toast222 = makeToast("222")
startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(Misc.mysteryLink)))
appVersionClick = 0
}
@@ -72,13 +64,16 @@ class SettingsFragment : Fragment() {
}
}
binding.appVersionText.text = BuildConfig.VERSION_NAME
@SuppressLint("SetTextI18n")
binding.appVersionText.text = "${BuildConfig.VERSION_NAME}-${if (BuildConfig.DEBUG) "debug" else "release"}"
}
private fun makeToast(text: String): Toast = Toast.makeText(context, text, Toast.LENGTH_SHORT)
override fun onDestroyView() {
super.onDestroyView()
_binding = null
}
private fun launchWebView() {
startActivity(Intent(context, FetcherActivity::class.java))
}
}
app/src/main/java/com/denizk0461/studip/model/CanteenOffer.kt
+23
View File
@@ -0,0 +1,23 @@
package com.denizk0461.studip.model
import androidx.room.Entity
import androidx.room.PrimaryKey
@Entity(tableName = "offers")
data class CanteenOffer(
@PrimaryKey val id: Int,
val date: String,
val category: String,
val title: String,
val price: String, // price for students
val isFair: Boolean, // artgerechte Tierhaltung
val isFish: Boolean, // Fisch
val isPoultry: Boolean, // Geflügel
val isLamb: Boolean, // Lamm
val isVital: Boolean, // mensaVital
val isBeef: Boolean, // Rindfleisch
val isPork: Boolean, // Schweinefleisch
val isVegan: Boolean, // plant-based (vegan)
val isVegetarian: Boolean, // vegetarisch
val isGame: Boolean, // Wild
)
app/src/main/java/com/denizk0461/studip/model/StudIPEvent.kt
+1 -1
View File
@@ -1,7 +1,6 @@
package com.denizk0461.studip.model
import androidx.room.Entity
import androidx.room.Ignore
import androidx.room.PrimaryKey
@Entity(tableName = "events")
@@ -13,6 +12,7 @@ data class StudIPEvent(
val day: Int, // 0 = Monday, 1 = Tuesday, 2 = Wednesday, 3 = Thursday, 4 = Friday
val timeslotStart: String, // the time the course starts
val timeslotEnd: String, // the time the course ends
val colour: String = "000000", // user-defined colour, TODO implement this
) {
// @Ignore private val timeSlotStartTimes: List<String> = listOf("06:15", "08:15", "10:15", "12:15", "14:15", "16:15", "18:15", "20:15")
app/src/main/java/com/denizk0461/studip/viewmodel/CanteenViewModel.kt
+18
View File
@@ -0,0 +1,18 @@
package com.denizk0461.studip.viewmodel
import android.app.Application
import androidx.lifecycle.LiveData
import com.denizk0461.studip.data.StwParser
import com.denizk0461.studip.model.CanteenOffer
class CanteenViewModel(app: Application) : TemplateViewModel(app) {
private val parser = StwParser()
val allOffers: LiveData<List<CanteenOffer>> = repo.allOffers
fun insertOffers(offers: List<CanteenOffer>) { doAsync { repo.insertOffers(offers) }}
fun nukeOffers() { doAsync { repo.nukeOffers() }}
fun fetchOffers(onFinish: () -> Unit) { doAsync { parser.parse(onFinish) }}
}