Implemented canteen scraper/parser functionality

This commit is contained in:
denizk0461
2023-04-16 18:19:06 +02:00
parent 2117535f63
commit 5543eb4e32
15 changed files with 270 additions and 89 deletions
@@ -13,7 +13,7 @@ import com.denizk0461.studip.data.Dependencies
import com.denizk0461.studip.databinding.ActivityMainBinding import com.denizk0461.studip.databinding.ActivityMainBinding
import com.denizk0461.studip.db.EventRepository import com.denizk0461.studip.db.EventRepository
import com.denizk0461.studip.fragment.EventFragment import com.denizk0461.studip.fragment.EventFragment
import com.denizk0461.studip.fragment.FoodFragment import com.denizk0461.studip.fragment.CanteenFragment
import com.denizk0461.studip.fragment.SettingsFragment import com.denizk0461.studip.fragment.SettingsFragment
class MainActivity : AppCompatActivity() { class MainActivity : AppCompatActivity() {
@@ -21,7 +21,7 @@ class MainActivity : AppCompatActivity() {
private lateinit var appBarConfiguration: AppBarConfiguration private lateinit var appBarConfiguration: AppBarConfiguration
private lateinit var binding: ActivityMainBinding private lateinit var binding: ActivityMainBinding
private var currentFragment = FragmentType.SCHEDULE private var currentFragment = FragmentType.SCHEDULE
private val fragments = listOf(EventFragment(), FoodFragment(), SettingsFragment()) private val fragments = listOf(EventFragment(), CanteenFragment(), SettingsFragment())
override fun onCreate(savedInstanceState: Bundle?) { override fun onCreate(savedInstanceState: Bundle?) {
WindowCompat.setDecorFitsSystemWindows(window, false) WindowCompat.setDecorFitsSystemWindows(window, false)
@@ -48,11 +48,11 @@ class StudIPEventAdapter(
if (!isAnyCourseHighlighted && currentItem.isCurrentCourse(currentCalendar)) { // highlight if (!isAnyCourseHighlighted && currentItem.isCurrentCourse(currentCalendar)) { // highlight
isAnyCourseHighlighted = true isAnyCourseHighlighted = true
holder.binding.cardBackground.strokeColor = colorPrimary.data holder.binding.cardBackground.strokeColor = colorPrimary.data
holder.binding.cardBackground.strokeWidth = 6 holder.binding.cardBackground.strokeWidth = 6 // TODO replace this with proper float -> dp conversion
// R.color.list_card_selected) // R.color.list_card_selected)
} else { // un-highlight } else { // un-highlight
holder.binding.cardBackground.strokeColor = colorTextHint.data holder.binding.cardBackground.strokeColor = colorTextHint.data
holder.binding.cardBackground.strokeWidth = 3 holder.binding.cardBackground.strokeWidth = 3 // this should (?) be 1dp
} }
holder.binding.cardBackground.setOnClickListener { holder.binding.cardBackground.setOnClickListener {
@@ -0,0 +1,144 @@
package com.denizk0461.studip.data
import android.util.Log
import com.denizk0461.studip.model.CanteenOffer
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
// Parser class for the Studierendenwerk cafeteria plans
class StwParser {
private var id = 0
fun parse(onFinish: () -> Unit) {
id = 0
val links = mutableListOf<String>()
if (true) { links.add(urlUniMensa) }
// if (true) { links.add(urlCafeCentral) }
// if (true) { links.add(urlNW1) }
// if (true) { links.add(urlGW2) }
// if (true) { links.add(urlHSBNeustadt) }
// if (true) { links.add(urlHSBAirport) }
// if (true) { links.add(urlHSBWerder) }
// if (true) { links.add(urlHfK) }
// if (true) { links.add(urlMensaBHV) }
// if (true) { links.add(urlCafeBHV) }
Dependencies.repo.nukeOffers()
links.forEach { link ->
Dependencies.repo.insertOffers(parseFromPage(link))
}
}
private fun parseFromPage(url: String): List<CanteenOffer> {
val items = mutableListOf<CanteenOffer>()
var date = ""
var day = 0
val doc = Jsoup.connect(url).get()
doc.getElementsByClass("food-plan").forEach { dayPlan ->
val rawDate = doc.getElementsByClass("tabs")[0]
.getElementsByClass("tab-date")[day].text().split(" ")
date = "${rawDate[0]}${rawDate[1].monthToNumber()}."
day += 1
dayPlan.getElementsByClass("food-category").forEach { category ->
val categoryTitle = category.getElementsByClass("category-name")[0].text()
category
.getElementsByTag("tbody")[0]
.getElementsByTag("tr").forEach { element ->
val tableRows = element.getElementsByTag("td")
val prefs =
element.getElementsByClass("field field-name-field-food-types")[0]
val newItem = CanteenOffer(
id = id,
date = date,
category = categoryTitle,
title = tableRows[1].getFilteredText(),
price = tableRows.getTextOrEmpty(2),
isFair = prefs.isDietaryPreferenceMet(PREFERENCE_FAIR),
isFish = prefs.isDietaryPreferenceMet(PREFERENCE_FISH),
isPoultry = prefs.isDietaryPreferenceMet(PREFERENCE_POULTRY),
isLamb = prefs.isDietaryPreferenceMet(PREFERENCE_LAMB),
isVital = prefs.isDietaryPreferenceMet(PREFERENCE_VITAL),
isBeef = prefs.isDietaryPreferenceMet(PREFERENCE_BEEF),
isPork = prefs.isDietaryPreferenceMet(PREFERENCE_PORK),
isVegan = prefs.isDietaryPreferenceMet(PREFERENCE_VEGAN),
isVegetarian = prefs.isDietaryPreferenceMet(PREFERENCE_VEGETARIAN),
isGame = prefs.isDietaryPreferenceMet(PREFERENCE_GAME),
)
items.add(newItem)
id += 1
Log.d("eek!", newItem.toString())
}
}
}
return items
}
private fun Element.isDietaryPreferenceMet(preference: String): Boolean =
getElementsByAttributeValue("src", preference).isNotEmpty()
private fun Element.getFilteredText(): String {
var text = html()
while (text.contains("<sup>")) {
text = text.substring(0 until text.indexOf("<sup>")) + text.substring(text.indexOf("</sup>")+6 until text.length)
}
return text
}
private fun Elements.getTextOrEmpty(index: Int): String = try {
get(index).text()
} catch (e: java.lang.IndexOutOfBoundsException) {
""
}
private fun String.monthToNumber(): String = when (this) {
"Jan" -> "01"
"Feb" -> "02"
"Mar" -> "03"
"Apr" -> "04"
"Mai" -> "05"
"Jun" -> "06"
"Jul" -> "07"
"Aug" -> "08"
"Sep" -> "09"
"Okt" -> "10"
"Nov" -> "11"
"Dez" -> "12"
else -> "00" // shouldn't occur
}
private val PREFERENCE_FAIR = "https://www.stw-bremen.de/sites/default/files/images/pictograms/at_small.png"
private val PREFERENCE_FISH = "https://www.stw-bremen.de/sites/default/files/images/pictograms/fisch.png"
private val PREFERENCE_POULTRY = "https://www.stw-bremen.de/sites/default/files/images/pictograms/geflugel.png"
private val PREFERENCE_LAMB = "https://www.stw-bremen.de/sites/default/files/images/pictograms/lamm.png"
private val PREFERENCE_VITAL = "https://www.stw-bremen.de/sites/default/files/images/pictograms/mensa_vital.png"
private val PREFERENCE_BEEF = "https://www.stw-bremen.de/sites/default/files/images/pictograms/rindfleisch.png"
private val PREFERENCE_PORK = "https://www.stw-bremen.de/sites/default/files/images/pictograms/schwein.png"
private val PREFERENCE_VEGAN = "https://www.stw-bremen.de/sites/default/files/images/pictograms/mensa_vegan.png"
private val PREFERENCE_VEGETARIAN = "https://www.stw-bremen.de/sites/default/files/images/pictograms/vegetarisch.png"
private val PREFERENCE_GAME = "https://www.stw-bremen.de/sites/default/files/images/pictograms/wild.png"
private val urlUniMensa = "https://www.stw-bremen.de/de/mensa/uni-mensa"
private val urlCafeCentral = "https://www.stw-bremen.de/de/mensa/cafe-central"
private val urlNW1 = "https://www.stw-bremen.de/de/mensa/nw-1"
private val urlGW2 = "https://www.stw-bremen.de/de/cafeteria/gw2"
// private val urlGraz = ""
private val urlHSBNeustadt = "https://www.stw-bremen.de/de/mensa/neustadtswall"
private val urlHSBWerder = "https://www.stw-bremen.de/de/mensa/werderstra%C3%9Fe"
private val urlHSBAirport = "https://www.stw-bremen.de/de/mensa/airport"
private val urlHfK = "https://www.stw-bremen.de/de/mensa/interimsmensa-hfk"
private val urlMensaBHV = "https://www.stw-bremen.de/de/mensa/bremerhaven"
private val urlCafeBHV = "https://www.stw-bremen.de/de/cafeteria/bremerhaven"
}
@@ -4,6 +4,7 @@ import androidx.lifecycle.LiveData
import androidx.room.Dao import androidx.room.Dao
import androidx.room.Insert import androidx.room.Insert
import androidx.room.Query import androidx.room.Query
import com.denizk0461.studip.model.CanteenOffer
import com.denizk0461.studip.model.StudIPEvent import com.denizk0461.studip.model.StudIPEvent
@Dao @Dao
@@ -20,4 +21,13 @@ interface EventDAO {
@Query("DELETE FROM events") @Query("DELETE FROM events")
fun nukeEvents() fun nukeEvents()
@get:Query("SELECT * FROM offers ORDER BY id")
val allOffers: LiveData<List<CanteenOffer>>
@Insert
fun insertOffers(offers: List<CanteenOffer>)
@Query("DELETE from offers")
fun nukeOffers()
} }
@@ -4,9 +4,10 @@ import android.content.Context
import androidx.room.Database import androidx.room.Database
import androidx.room.Room import androidx.room.Room
import androidx.room.RoomDatabase import androidx.room.RoomDatabase
import com.denizk0461.studip.model.CanteenOffer
import com.denizk0461.studip.model.StudIPEvent import com.denizk0461.studip.model.StudIPEvent
@Database(entities = [StudIPEvent::class], version = 3) @Database(entities = [StudIPEvent::class, CanteenOffer::class], version = 6)
abstract class EventDatabase : RoomDatabase() { abstract class EventDatabase : RoomDatabase() {
abstract fun dao(): EventDAO abstract fun dao(): EventDAO
@@ -2,6 +2,7 @@ package com.denizk0461.studip.db
import android.app.Application import android.app.Application
import androidx.lifecycle.LiveData import androidx.lifecycle.LiveData
import com.denizk0461.studip.model.CanteenOffer
import com.denizk0461.studip.model.StudIPEvent import com.denizk0461.studip.model.StudIPEvent
class EventRepository(app: Application) { class EventRepository(app: Application) {
@@ -10,7 +11,12 @@ class EventRepository(app: Application) {
val allEvents: LiveData<List<StudIPEvent>> = dao.allEvents val allEvents: LiveData<List<StudIPEvent>> = dao.allEvents
fun nukeEvents() { dao.nukeEvents() }
fun insertEvent(event: StudIPEvent) { dao.insertEvent(event) } fun insertEvent(event: StudIPEvent) { dao.insertEvent(event) }
fun insertEvents(events: List<StudIPEvent>) { dao.insertEvents(events) } fun insertEvents(events: List<StudIPEvent>) { dao.insertEvents(events) }
fun nukeEvents() { dao.nukeEvents() }
val allOffers: LiveData<List<CanteenOffer>> = dao.allOffers
fun insertOffers(offers: List<CanteenOffer>) { dao.insertOffers(offers) }
fun nukeOffers() { dao.nukeOffers() }
} }
@@ -0,0 +1,46 @@
package com.denizk0461.studip.fragment
import android.os.Bundle
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import androidx.fragment.app.Fragment
import androidx.fragment.app.viewModels
import com.denizk0461.studip.databinding.FragmentCanteenBinding
import com.denizk0461.studip.viewmodel.CanteenViewModel
class CanteenFragment : Fragment() {
private var _binding: FragmentCanteenBinding? = null
// This property is only valid between onCreateView and
// onDestroyView.
private val binding get() = _binding!!
private val viewModel: CanteenViewModel by viewModels()
override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
_binding = FragmentCanteenBinding.inflate(inflater, container, false)
return binding.root
}
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
viewModel.allOffers.observe(viewLifecycleOwner) { offers ->
// TODO this must be changed. if an exception is raised, then this will not fire
binding.swipeRefreshLayout.isRefreshing = false
}
binding.swipeRefreshLayout.setOnRefreshListener {
viewModel.fetchOffers {
// binding.swipeRefreshLayout.isRefreshing = false
}
}
}
override fun onDestroyView() {
super.onDestroyView()
_binding = null
}
}
@@ -1,15 +1,12 @@
package com.denizk0461.studip.fragment package com.denizk0461.studip.fragment
import android.content.Intent
import android.os.Bundle import android.os.Bundle
import androidx.fragment.app.Fragment import androidx.fragment.app.Fragment
import android.view.LayoutInflater import android.view.LayoutInflater
import android.view.View import android.view.View
import android.view.ViewGroup import android.view.ViewGroup
import androidx.fragment.app.viewModels import androidx.fragment.app.viewModels
import androidx.navigation.fragment.NavHostFragment.Companion.findNavController
import com.denizk0461.studip.R import com.denizk0461.studip.R
import com.denizk0461.studip.activity.FetcherActivity
import com.denizk0461.studip.adapter.StudIPEventAdapter import com.denizk0461.studip.adapter.StudIPEventAdapter
import com.denizk0461.studip.adapter.StudIPEventPageAdapter import com.denizk0461.studip.adapter.StudIPEventPageAdapter
import com.denizk0461.studip.databinding.FragmentEventBinding import com.denizk0461.studip.databinding.FragmentEventBinding
@@ -59,7 +56,7 @@ class EventFragment : Fragment() {
viewPagerAdapter = StudIPEventPageAdapter(listOf(), object : StudIPEventAdapter.OnClickListener { viewPagerAdapter = StudIPEventPageAdapter(listOf(), object : StudIPEventAdapter.OnClickListener {
override fun onClick(event: StudIPEvent) { override fun onClick(event: StudIPEvent) {
// do nothing
} }
override fun onLongClick(event: StudIPEvent) { override fun onLongClick(event: StudIPEvent) {
@@ -76,10 +73,6 @@ class EventFragment : Fragment() {
switchToCurrentDayView() switchToCurrentDayView()
} }
binding.fab.setOnClickListener { view ->
launchWebview()
}
// binding.buttonFirst.setOnClickListener { // binding.buttonFirst.setOnClickListener {
// findNavController().navigate(R.id.action_FirstFragment_to_SecondFragment) // findNavController().navigate(R.id.action_FirstFragment_to_SecondFragment)
// } // }
@@ -99,16 +92,4 @@ class EventFragment : Fragment() {
binding.viewPager.currentItem = dayOfWeek binding.viewPager.currentItem = dayOfWeek
// binding.dayTabLayout.getTabAt(dayOfWeek)?.select() // binding.dayTabLayout.getTabAt(dayOfWeek)?.select()
} }
private fun launchWebview() {
// findNavController(this).navigate(R.id.action_FirstFragment_to_SecondFragment)
startActivity(Intent(context, FetcherActivity::class.java))
}
/**
* Highlights the current or next course.
*/
private fun highlightCurrentCourse() {
}
} }
@@ -1,32 +0,0 @@
package com.denizk0461.studip.fragment
import android.os.Bundle
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import androidx.fragment.app.Fragment
import com.denizk0461.studip.databinding.FragmentFoodBinding
class FoodFragment : Fragment() {
private var _binding: FragmentFoodBinding? = null
// This property is only valid between onCreateView and
// onDestroyView.
private val binding get() = _binding!!
override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
_binding = FragmentFoodBinding.inflate(inflater, container, false)
return binding.root
}
override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState)
}
override fun onDestroyView() {
super.onDestroyView()
_binding = null
}
}
@@ -1,8 +1,8 @@
package com.denizk0461.studip.fragment package com.denizk0461.studip.fragment
import android.annotation.SuppressLint
import android.content.Intent import android.content.Intent
import android.net.Uri import android.net.Uri
import android.os.Build
import android.os.Bundle import android.os.Bundle
import android.view.LayoutInflater import android.view.LayoutInflater
import android.view.View import android.view.View
@@ -10,6 +10,7 @@ import android.view.ViewGroup
import android.widget.Toast import android.widget.Toast
import androidx.fragment.app.Fragment import androidx.fragment.app.Fragment
import com.denizk0461.studip.BuildConfig import com.denizk0461.studip.BuildConfig
import com.denizk0461.studip.activity.FetcherActivity
import com.denizk0461.studip.data.Misc import com.denizk0461.studip.data.Misc
import com.denizk0461.studip.databinding.FragmentSettingsBinding import com.denizk0461.studip.databinding.FragmentSettingsBinding
@@ -31,15 +32,15 @@ class SettingsFragment : Fragment() {
override fun onViewCreated(view: View, savedInstanceState: Bundle?) { override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
super.onViewCreated(view, savedInstanceState) super.onViewCreated(view, savedInstanceState)
var toast2 = makeToast("2")
var toast22 = makeToast("22")
var toast222 = makeToast("222")
// binding.layoutQuarter.setOnClickListener { // binding.layoutQuarter.setOnClickListener {
// isQuarterChecked = !isQuarterChecked // isQuarterChecked = !isQuarterChecked
// binding.switchQuarter.isChecked = isQuarterChecked // binding.switchQuarter.isChecked = isQuarterChecked
// } // }
binding.buttonRefreshSchedule.setOnClickListener {
launchWebView()
}
binding.switchQuarter.setOnClickListener { binding.switchQuarter.setOnClickListener {
isQuarterChecked = !isQuarterChecked isQuarterChecked = !isQuarterChecked
} }
@@ -47,24 +48,15 @@ class SettingsFragment : Fragment() {
binding.buttonAppVersion.setOnClickListener { binding.buttonAppVersion.setOnClickListener {
when (appVersionClick) { when (appVersionClick) {
19 -> { 19 -> {
toast2.show()
appVersionClick += 1 appVersionClick += 1
} }
20 -> { 20 -> {
toast2.cancel()
toast22.show()
appVersionClick += 1 appVersionClick += 1
} }
21 -> { 21 -> {
toast22.cancel()
toast222.show()
appVersionClick += 1 appVersionClick += 1
} }
22 -> { 22 -> {
toast222.cancel()
toast2 = makeToast("2")
toast22 = makeToast("22")
toast222 = makeToast("222")
startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(Misc.mysteryLink))) startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(Misc.mysteryLink)))
appVersionClick = 0 appVersionClick = 0
} }
@@ -72,13 +64,16 @@ class SettingsFragment : Fragment() {
} }
} }
binding.appVersionText.text = BuildConfig.VERSION_NAME @SuppressLint("SetTextI18n")
binding.appVersionText.text = "${BuildConfig.VERSION_NAME}-${if (BuildConfig.DEBUG) "debug" else "release"}"
} }
private fun makeToast(text: String): Toast = Toast.makeText(context, text, Toast.LENGTH_SHORT)
override fun onDestroyView() { override fun onDestroyView() {
super.onDestroyView() super.onDestroyView()
_binding = null _binding = null
} }
private fun launchWebView() {
startActivity(Intent(context, FetcherActivity::class.java))
}
} }
@@ -0,0 +1,23 @@
package com.denizk0461.studip.model
import androidx.room.Entity
import androidx.room.PrimaryKey
@Entity(tableName = "offers")
data class CanteenOffer(
@PrimaryKey val id: Int,
val date: String,
val category: String,
val title: String,
val price: String, // price for students
val isFair: Boolean, // artgerechte Tierhaltung
val isFish: Boolean, // Fisch
val isPoultry: Boolean, // Geflügel
val isLamb: Boolean, // Lamm
val isVital: Boolean, // mensaVital
val isBeef: Boolean, // Rindfleisch
val isPork: Boolean, // Schweinefleisch
val isVegan: Boolean, // plant-based (vegan)
val isVegetarian: Boolean, // vegetarisch
val isGame: Boolean, // Wild
)
@@ -1,7 +1,6 @@
package com.denizk0461.studip.model package com.denizk0461.studip.model
import androidx.room.Entity import androidx.room.Entity
import androidx.room.Ignore
import androidx.room.PrimaryKey import androidx.room.PrimaryKey
@Entity(tableName = "events") @Entity(tableName = "events")
@@ -13,6 +12,7 @@ data class StudIPEvent(
val day: Int, // 0 = Monday, 1 = Tuesday, 2 = Wednesday, 3 = Thursday, 4 = Friday val day: Int, // 0 = Monday, 1 = Tuesday, 2 = Wednesday, 3 = Thursday, 4 = Friday
val timeslotStart: String, // the time the course starts val timeslotStart: String, // the time the course starts
val timeslotEnd: String, // the time the course ends val timeslotEnd: String, // the time the course ends
val colour: String = "000000", // user-defined colour, TODO implement this
) { ) {
// @Ignore private val timeSlotStartTimes: List<String> = listOf("06:15", "08:15", "10:15", "12:15", "14:15", "16:15", "18:15", "20:15") // @Ignore private val timeSlotStartTimes: List<String> = listOf("06:15", "08:15", "10:15", "12:15", "14:15", "16:15", "18:15", "20:15")
@@ -0,0 +1,18 @@
package com.denizk0461.studip.viewmodel
import android.app.Application
import androidx.lifecycle.LiveData
import com.denizk0461.studip.data.StwParser
import com.denizk0461.studip.model.CanteenOffer
class CanteenViewModel(app: Application) : TemplateViewModel(app) {
private val parser = StwParser()
val allOffers: LiveData<List<CanteenOffer>> = repo.allOffers
fun insertOffers(offers: List<CanteenOffer>) { doAsync { repo.insertOffers(offers) }}
fun nukeOffers() { doAsync { repo.nukeOffers() }}
fun fetchOffers(onFinish: () -> Unit) { doAsync { parser.parse(onFinish) }}
}
@@ -27,15 +27,4 @@
app:layout_constraintBottom_toBottomOf="parent"/> app:layout_constraintBottom_toBottomOf="parent"/>
<!-- app:layout_behavior="@string/appbar_scrolling_view_behavior"--> <!-- app:layout_behavior="@string/appbar_scrolling_view_behavior"-->
<com.google.android.material.floatingactionbutton.FloatingActionButton
android:id="@+id/fab"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
android:layout_marginEnd="@dimen/fab_margin"
android:layout_marginBottom="@dimen/fab_margin"
app:srcCompat="@drawable/refresh"
app:backgroundTint="?attr/colorPrimary"/>
</androidx.constraintlayout.widget.ConstraintLayout> </androidx.constraintlayout.widget.ConstraintLayout>
+2 -2
View File
@@ -17,8 +17,8 @@
<color name="colorTextHintDay">#2A2828</color> <color name="colorTextHintDay">#2A2828</color>
<color name="colorTextHintNight">#B9B6B6</color> <color name="colorTextHintNight">#B9B6B6</color>
<color name="colorTextHintLighterDay">#3E3C3C</color> <color name="colorTextHintLighterDay">#AFACAC</color>
<color name="colorTextHintLighterNight">#918D8D</color> <color name="colorTextHintLighterNight">#5E5A5A</color>
<color name="colorBackgroundDay">#EBEBEB</color> <color name="colorBackgroundDay">#EBEBEB</color>
<color name="colorBackgroundNight">#151414</color> <color name="colorBackgroundNight">#151414</color>