Flask app working really slow with opencv

Question

I have a flask application which reads frame from camera and streams it to the website.

Camera.py

from threading import Thread
from copy import deepcopy

import queue
import cv2

class Camera(Thread):
    def __init__(self, cam, normalQue, detectedQue):
        Thread.__init__(self)
        self.__cam = cam
        self.__normalQue = normalQue
        self.__detectedQue = detectedQue
        self.__shouldStop = False
        
    def __del__(self):
        self.__cam.release()
        print('Camera released')
        
    def run(self):
        while True:
            rval, frame = self.__cam.read()

            if rval:
                frame = cv2.resize(frame, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
                _, jpeg = cv2.imencode('.jpg', frame)

                self.__normalQue.put(jpeg.tobytes())
                self.__detectedQue.put(deepcopy(jpeg.tobytes()))

            if self.__shouldStop:
                break

    def stopCamera(self):
        self.__shouldStop = True

From what you can see I am just reading the frame, resizing it and storing in two different ques. Nothing too complex. I also have two two classes responsible for mjpeg stream:

NormalVideoStream.py

from threading import Thread

import traceback
import cv2

class NormalVideoStream(Thread):
    def __init__(self, framesQue):
        Thread.__init__(self)
        self.__frames = framesQue
        self.__img = None

    def run(self):
        while True:
            if self.__frames.empty():
                continue

            self.__img = self.__frames.get()

    def gen(self):
        while True:
            try:
                if self.__img is None:
                    print('Normal stream frame is none')
                    continue

                yield (b'--frame\r\n'
                    b'Content-Type: image/jpeg\r\n\r\n' + self.__img + b'\r\n')
            except:
                traceback.print_exc()
                print('Normal video stream genenation exception')

and

DetectionVideoStream.py

from threading import Thread

import cv2
import traceback

class DetectionVideoStream(Thread):
    def __init__(self, framesQue):
        Thread.__init__(self)
        
        self.__frames = framesQue
        self.__img = None
        self.__faceCascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')

    def run(self):
        while True:
            if self.__frames.empty():
                continue
            
            self.__img = self.__detectFace()

    def gen(self):
        while True:
            try:
                if self.__img is None:
                    print('Detected stream frame is none')

                yield (b'--frame\r\n'
                    b'Content-Type: image/jpeg\r\n\r\n' + self.__img + b'\r\n')
            except:
                traceback.print_exc()
                print('Detection video stream genenation exception')
    
    def __detectFace(self):
        retImg = None

        try:
            img = self.__frames.get()

            gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

            faces = self.__faceCascade.detectMultiScale(gray, 1.1, 4)

            for (x, y, w, h) in faces:
                cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)

            (_, encodedImage) = cv2.imencode('.jpg', img)

            retImg = encodedImage.tobytes()
        except:
            traceback.print_exc()
            print('Face detection exception')

        return retImg

From what you can see in both streams I am reading camera frames from ques in infinite loop. Both classes also have gen() method which generates frame to site itself. Only difference is that in detection stream I am also doing face recognition.

Now in my main file:

main.py

from flask import Blueprint, render_template, Response, abort, redirect, url_for
from flask_login import login_required, current_user
from queue import Queue
from . import db
from .Camera import Camera
from .NormalVideoStream import NormalVideoStream
from .DetectionVideoStream import DetectionVideoStream
from .models import User

import cv2

main = Blueprint('main', __name__)

# Queues for both streams
framesNormalQue = Queue(maxsize=0)
framesDetectionQue = Queue(maxsize=0)
print('Queues created')

# RPi camera instance
camera = Camera(cv2.VideoCapture(0), framesNormalQue, framesDetectionQue)
camera.start()
print('Camera thread started')

# Streams
normalStream = NormalVideoStream(framesNormalQue)
detectionStream = DetectionVideoStream(framesDetectionQue)
print('Streams created')

normalStream.start()
print('Normal stream thread started')
detectionStream.start()
print('Detection stream thread started')

@main.route('/')
def index():
    return render_template('index.html')

@main.route('/profile', methods=["POST", "GET"])
def profile():
    if not current_user.is_authenticated:
        abort(403)

    return render_template('profile.html', name=current_user.name, id=current_user.id, detectionState=current_user.detectionState)

@main.route('/video_stream/<int:stream_id>')
def video_stream(stream_id):
    if not current_user.is_authenticated:
        abort(403)

    print(f'Current user detection: {current_user.detectionState}')

    global detectionStream
    global normalStream

    stream = None

    if current_user.detectionState:
        stream = detectionStream
        print('Stream set to detection one')
    else:
        stream = normalStream
        print('Stream set to normal one')

    return Response(stream.gen(), mimetype='multipart/x-mixed-replace; boundary=frame')

@main.route('/detection')
def detection():
    if not current_user.is_authenticated:
        abort(403)

    if current_user.detectionState:
        current_user.detectionState = False
    else:
        current_user.detectionState = True

    user = User.query.filter_by(id=current_user.id)
    user.detectionState = current_user.detectionState

    db.session.commit()

    return redirect(url_for('main.profile', id=current_user.id, user_name=current_user.name))

@main.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

@main.errorhandler(403)
def page_forbidden(e):
    return render_template('403.html'), 403

I am creating camera, ques and streams object globally. Also when user logs in on website, he will be able to see live video stream. There is also a button which changes the stream which is currently presented.

Whole project is working well with one exception: when I change stream to detection one it will have huge lag (around 10/15 seconds) which makes whole thing unfunctional. Tried to search a bug/optimalization in my own but can't find anything. On purpose I am running everything on separate threads to unoverload app but it looks like this is not enought. Lag on a level of 1 - 2 seconds will be acceptable, but not 10+. So guys, maybe you can see some bug here? Or know how to optimalize it?

Also need to mention that whole app is running on RPi 4B 4GB and I am accessing website on my desktop. Default server is changed to the Nginx and Gunicorn. From what I can see Pi's CPU usage is 100% when app is working. When testing on default server behaviout is the same. Guess that 1,5 GHz CPU have enough power to run it more smoothly.

Answer 1

• One option is using VideoStream

• The reason VideoCapture is so slow because the VideoCapture pipeline spends the most time on the reading and decoding the next frame. While the next frame is being read, decode, and returned the OpenCV application is completely blocked.

• VideoStream solves the problem by using a queue structure, concurrently read, decode, and return the current frame.

• VideoStream supports both PiCamera and webcam.

All you need to is:

• 1. Install imutils:

---

• For virtual environment: pip install imutils

  ---

• For anaconda environment: conda install -c conda-forge imutils

• 2. Initialize VideoStream on main.py

---

• import time
  from imutils.video import VideoStream
  
  vs = VideoStream(usePiCamera=True).start()  # For PiCamera
  # vs = VideoStream(usePiCamera=False).start() # For Webcam
  
  camera = Camera(vs, framesNormalQue, framesDetectionQue)
  

• 3. In your Camera.py

---

• In run(self) method:

---

* ```python
  def run(self):
      while True:
          frame = self.__cam.read()
          frame = cv2.resize(frame, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
            _, jpeg = cv2.imencode('.jpg', frame)

            self.__normalQue.put(jpeg.tobytes())
            self.__detectedQue.put(deepcopy(jpeg.tobytes()))

        if self.__shouldStop:
            break
  ```

Answer 2

I'm not really surprised about your problem, in general "detection" using a lot of your computation time, because performing a cascaded classification algorithm is a demanding computational task. I found a source which compares cascaded classification algos for there performance link

An easy solution, would be to reduce the frame rate, when processing your detection. An easy implementation to reduce performance demand could be something like a skip counter e.g.

frameSkipFactor = 3 # use every third frame
frameCounter = 0

if (frameCounter%frameSkipFactor==0):
         #process
else:
         print("ignore frame", frameCounter)

frameCounter+=1

Nevertheless you will have a lag, because the detection calculation will produce always a time offset. I you planning to construct a "real time" classification camera system, please look for another class of classification algos, which are more designed for this use-case. I followed an discussion here: real time class algos

Another solution could be using a bigger "hardware hammer" than the rpi e.g. an GPU implementation of the algo via Cuda etc.

Answer 3

One of the issue which even I had was regrading the encoding and decoding. Like the encoder of Opencv is too slow so try to use encoder from simplejpeg. Use pip3 install simplejpeg and with respect to using cv2.imencode() use simplejpeg.encode_jpeg()

Answer 4

What I have found that main reason of getting slow frames is that you have higher resolution and frame rates.

To tackle this problem you can change the resolution to something 640 width by 480 height with fps 30 or less upto 5 fps (if you only need face detection) and can implement resizing of OpenCV (cv2.resize() function) by 0.25 resizing factor of fx and fy. Do this if you don't want higher resolution streams. Works smoothly with opencv. I wanted to try out that VideoStream code (from imutils) as it is also used by Adrian Rosebrock of PyImageSearch. I will use in later projects.

For reference, I am posting code snippet in the following. Special Thanks to Adrian Rosebrock and ageitey face_recognition as their code helped me making it.

class Camera(object):

    SHRINK_RATIO = 0.25
    FPS = 5
    FRAME_RATE = 5
    WIDTH = 640
    HEIGHT = 480

def __init__(self):
    """ initializing camera with settings """
    self.cap = cv2.VideoCapture(0)
    self.cap.set(3,Camera.WIDTH)
    self.cap.set(4,Camera.HEIGHT)
    self.cap.set(5,Camera.FPS)
    self.cap.set(7,Camera.FRAME_RATE)

def get_frame(self):
""" get frames from the camera """
success, frame = self.cap.read()
    
    if success == True:
       
        # Resizing to 0.25 scale
        
        rescale_frame = cv2.resize(frame, None, fx= Camera.SHRINK_RATIO, fy=Camera.SHRINK_RATIO)
        cascPath = "haarcascade_frontalface_default.xml"
        faceCascade = cv2.CascadeClassifier(cascPath)
        gray = cv2.cvtColor(rescale_frame, cv2.COLOR_BGR2GRAY)

        # detecting faces 
        faces = faceCascade.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30,30)
        )

        # Draw a rectangle around the faces
        if len(faces) != 0:
            for (x, y, w, h) in faces:
                x *=4
                y *=4
                w *=4
                h *=4
                # Draw rectangle across faces in current frame
                cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
        # return frame outside if-statement
        return frame

Also keep in mind that JPEG is the accelerated codec, use:

cv2.imencode(".jpg",frame)[1].tobytes()