[Spring boot, Vue3] 사이트 크롤링 하기

채용공고 크롤링

가져올 URL : https://job.incruit.com/jobdb_list/searchjob.asp?occ2=574&occ1=120&rgn2=11

---

 

 

build.gradle에 jsoup 추가하기

implementation 'org.jsoup:jsoup:1.14.3'

CrawlingPage.vue

<template>
    <div>
      <MenuPage />
      <div class="boundary">
        <h1 class="title">취업 리스트 크롤링</h1>
        <jobSecMenu />
        <div class="container">
          <div v-if="jobs.length">
            <table class="job-table">
              <thead>
                <tr>
                  <th>채용 제목</th>
                  <th>기관</th>
                  <th>고용 형태</th>
                  <th>근무 지역</th>
                  <th>접수 기간</th>
                  <th>상세 정보</th>
                </tr>
              </thead>
              <tbody>
                <tr v-for="job in jobs" :key="job.detailLink">
                  <td>{{ job.title }}</td>
                  <td>{{ job.company }}</td>
                  <td>{{ job.location }}</td>
                  <td>{{ job.date }}</td>
                  <td><a :href="job.detailLink" target="_blank">상세보기</a></td>
                </tr>
              </tbody>
            </table>
          </div>
          <div v-else>
            <p>채용 정보를 불러오는 중...</p>
          </div>
        </div>
      </div>
    </div>
  </template>

  <script>
  import MenuPage from "@/components/MenuPage.vue";
  import jobSecMenu from "@/views/jobs/jobSecMenu.vue";
  import axios from 'axios';

  export default {
    name: "Crawling",
    components: {
      MenuPage,
      jobSecMenu,
    },
    data() {
      return {
        jobs: []
      };
    },
    methods: {
      fetchJobs() {
        axios.get('http://localhost:3000/api/jobslists')
          .then(response => {
            this.jobs = response.data;
          })
          .catch(error => {
            console.error('Error fetching jobs:', error);
          });
      }
    },
    created() {
      this.fetchJobs();
    }
  };
  </script>

  <style scoped>
  .container {
    display: flex;
    justify-content: center;
    align-items: center;
  }

  .job-table {
    width: 100%;
    border-collapse: collapse;
  }

  .job-table th, .job-table td {
    border: 1px solid #ddd;
    padding: 8px;
    text-align: left;
  }

  .job-table th {
    background-color: #f2f2f2;
  }

  .job-table a {
    color: blue;
    text-decoration: underline;
  }

  .job-table td:last-child {
    width: 90px; /* 상세 정보 칸 크기 조정 */
    word-wrap: break-word; /* 단어가 길 경우 줄 바꿈 처리 */
  }
  </style>

JobslistsController.java

package com.kidwiz.web.controller;

import com.kidwiz.web.DTO.JoblistsDTO;
import com.kidwiz.web.service.JoblistsService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.io.IOException;
import java.util.List;

@RestController
@RequestMapping(value="/api/jobslists")
public class JoblistsController {

    @Autowired
    private JoblistsService joblistsService;

    @GetMapping
    public List<JoblistsDTO> getJobs() {
        try {
            return joblistsService.getJobs();
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }
}

JoblistsService.java

크롤링 할 페이지의 코드를 확인한다. html형식을 알아야 불러올 수 있음.

 

---

package com.kidwiz.web.service;

import com.kidwiz.web.DTO.JoblistsDTO;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

@Service
public class JoblistsService {
    public List<JoblistsDTO> getJobs() throws IOException {
        String url = "https://job.incruit.com/jobdb_list/searchjob.asp?occ2=574&occ1=120&rgn2=11";
        Document document = Jsoup.connect(url).get();

        List<JoblistsDTO> jobs = new ArrayList<>();
        Elements jobElements = document.select(".c_row"); // 공고 리스트 선택

        for (Element jobElement : jobElements) {
            String title = jobElement.select(".cell_mid .cl_top a").text();  // 채용 제목
            String company = jobElement.select(".cell_first .cpname").text();  // 회사 이름
            String location = jobElement.select(".cell_mid .cl_md span:nth-of-type(3)").text();  // 위치
            String date = jobElement.select(".cell_last .cl_btm span:nth-of-type(1)").text();  // 마감 날짜
            String detailLink = jobElement.select(".cell_mid .cl_top a").attr("href");  // 자세히 보기 링크

            // Debug logging
            System.out.println("Title: " + title);
            System.out.println("Company: " + company);
            System.out.println("Location: " + location);
            System.out.println("Date: " + date);
            System.out.println("Detail Link: " + detailLink);

            JoblistsDTO job = new JoblistsDTO(title, company, location, date, detailLink);
            jobs.add(job);
        }

        return jobs;
    }
}

JoblistsDTO.java

package com.kidwiz.web.DTO;

import lombok.Data;

@Data
public class JoblistsDTO {

    private String title;
    private String company;
    private String location;
    private String date;
    private String detailLink;

    public JoblistsDTO(String title, String company, String location, String date, String detailLink) {
        this.title = title;
        this.company = company;
        this.location = location;
        this.date = date;
        this.detailLink = detailLink;
    }
}