webrtc test

README.md

@@ -37,3 +37,14 @@ sudo apt install gstreamer1.0-tools
 Sender (Pi): gst-launch-1.0 -e v4l2src do-timestamp=true ! video/x-h264,width=640,height=480,framerate=30/1 ! h264parse ! rtph264pay config-interval=1 ! gdppay ! udpsink host=192.168.178.20 port=5000
 
 Receiver: gst-launch-1.0 -v udpsrc port=5000 ! gdpdepay ! rtph264depay ! avdec_h264 ! fpsdisplaysink sync=false text-overlay=false
+
+## documentation
+
+https://flask-socketio.readthedocs.io/en/latest/
+
+https://github.com/pfertyk/webrtc-working-example
+
+https://github.com/nanomosfet/WebRTC-Flask-server/blob/master/webRTCserver/webRTCserver.py
+
+http://blog.nirbheek.in/2018/02/gstreamer-webrtc.html
+https://gitlab.freedesktop.org/gstreamer/gst-examples/-/tree/master/webrtc/sendrecv/gst

raspberry/requirements.py

@@ -8,3 +8,6 @@ python3-picamera
 libjs-jquery
 libjs-bootstrap
 python3-eventlet
+
+gir1.2-gst-plugins-bad-1.0
+python3-gst-1.0

raspberry/roberto/__init__.py

@@ -18,6 +18,8 @@ login.login_view = "users.login"
 socketio = SocketIO()
 from roberto.camera.camera_opencv import Camera
 camera = Camera()
+from roberto.camera.camera_gstreamer_webrtc import WebRTCCamera
+webrtccamera = WebRTCCamera(1000, 1001)
 from roberto.Serial import Serial
 serial = Serial()
 

raspberry/roberto/camera/camera_gstreamer_webrtc.py

@@ -0,0 +1,150 @@
+import os
+import sys
+
+import gi
+gi.require_version('Gst', '1.0')
+from gi.repository import Gst
+gi.require_version('GstWebRTC', '1.0')
+from gi.repository import GstWebRTC
+gi.require_version('GstSdp', '1.0')
+from gi.repository import GstSdp
+
+import json
+
+PIPELINE_DESC = '''
+webrtcbin name=sendrecv bundle-policy=max-bundle stun-server=stun://stun.l.google.com:19302
+ videotestsrc is-live=true pattern=ball ! videoconvert ! queue ! vp8enc deadline=1 ! rtpvp8pay !
+ queue ! application/x-rtp,media=video,encoding-name=VP8,payload=97 ! sendrecv.
+ audiotestsrc is-live=true wave=red-noise ! audioconvert ! audioresample ! queue ! opusenc !rtpopuspay !
+ queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 ! sendrecv.
+'''
+
+class WebRTCCamera:
+    def __init__(self, id_, peer_id, server=None):
+        self.id_ = id_
+        self.conn = None
+        self.pipe = None
+        self.webrtc = None
+        self.socketio = None
+        self.peer_id = peer_id
+        self.room = 'default'
+        self.sid = 'gstwebrtc1000'
+        self.server = server or 'wss://localhost:5000/webrtc'
+        self.connected = False
+
+        Gst.init(None)
+        if not check_plugins():
+            sys.exit(1)
+
+    def connect(self, socketio, room, sid):
+        self.socketio = socketio
+        self.room = room
+        self.sid = sid
+        self.start_pipeline()
+        self.connected = True
+
+    def disconnect(self):
+        self.connected = False
+        self.close_pipeline()
+        self.socketio = None
+
+    def start_pipeline(self):
+        self.pipe = Gst.parse_launch(PIPELINE_DESC)
+        self.webrtc = self.pipe.get_by_name('sendrecv')
+        self.webrtc.connect('on-negotiation-needed', self.on_negotiation_needed)
+        self.webrtc.connect('on-ice-candidate', self.send_ice_candidate_message)
+        self.webrtc.connect('pad-added', self.on_incoming_stream)
+        self.pipe.set_state(Gst.State.PLAYING)
+
+    def close_pipeline(self):
+        self.pipe.set_state(Gst.State.NULL)
+        self.pipe = None
+        self.webrtc = None
+
+    def handle_sdp_answer(self, sdp):
+        print ('Received answer:\n%s' % sdp)
+        res, sdpmsg = GstSdp.SDPMessage.new()
+        GstSdp.sdp_message_parse_buffer(bytes(sdp.encode()), sdpmsg)
+        answer = GstWebRTC.WebRTCSessionDescription.new(GstWebRTC.WebRTCSDPType.ANSWER, sdpmsg)
+        promise = Gst.Promise.new()
+        self.webrtc.emit('set-remote-description', answer, promise)
+        promise.interrupt()
+
+    def handle_ice(self, ice):
+        candidate = ice['candidate']
+        sdpmlineindex = ice['sdpMLineIndex']
+        self.webrtc.emit('add-ice-candidate', sdpmlineindex, candidate)
+
+    def on_negotiation_needed(self, element):
+        promise = Gst.Promise.new_with_change_func(self.on_offer_created, element, None)
+        element.emit('create-offer', None, promise)
+
+    def send_ice_candidate_message(self, _, mlineindex, candidate):
+        icemsg = json.dumps({'type': 'candidate', 'candidate': {'candidate': candidate, 'sdpMLineIndex': mlineindex}})
+        self.socketio.emit('data', type='candidate', data=icemsg, room=self.room, namespace='/webrtc', skip_sid=self.sid)
+
+    def on_incoming_stream(self, _, pad):
+        if pad.direction != Gst.PadDirection.SRC:
+            return
+
+        decodebin = Gst.ElementFactory.make('decodebin')
+        decodebin.connect('pad-added', self.on_incoming_decodebin_stream)
+        self.pipe.add(decodebin)
+        decodebin.sync_state_with_parent()
+        self.webrtc.link(decodebin)
+
+    def on_incoming_decodebin_stream(self, _, pad):
+        if not pad.has_current_caps():
+            print (pad, 'has no caps, ignoring')
+            return
+
+        caps = pad.get_current_caps()
+        assert (len(caps))
+        s = caps[0]
+        name = s.get_name()
+        if name.startswith('video'):
+            q = Gst.ElementFactory.make('queue')
+            conv = Gst.ElementFactory.make('videoconvert')
+            sink = Gst.ElementFactory.make('autovideosink')
+            self.pipe.add(q, conv, sink)
+            self.pipe.sync_children_states()
+            pad.link(q.get_static_pad('sink'))
+            q.link(conv)
+            conv.link(sink)
+        elif name.startswith('audio'):
+            q = Gst.ElementFactory.make('queue')
+            conv = Gst.ElementFactory.make('audioconvert')
+            resample = Gst.ElementFactory.make('audioresample')
+            sink = Gst.ElementFactory.make('autoaudiosink')
+            self.pipe.add(q, conv, resample, sink)
+            self.pipe.sync_children_states()
+            pad.link(q.get_static_pad('sink'))
+            q.link(conv)
+            conv.link(resample)
+            resample.link(sink)
+
+    def on_offer_created(self, promise, _, __):
+        promise.wait()
+        reply = promise.get_reply()
+        offer = reply['offer']
+        promise = Gst.Promise.new()
+        self.webrtc.emit('set-local-description', offer, promise)
+        promise.interrupt()
+        text = offer.sdp.as_text()
+        print ('Sending offer:\n%s' % text)
+        msg = json.dumps({'type': 'offer', 'sdp': text})
+        self.socketio.emit('data', type='offer', data=msg, room=self.room, namespace='/webrtc', skip_sid=self.sid)
+
+
+
+
+
+
+def check_plugins():
+    needed = ["opus", "vpx", "nice", "webrtc", "dtls", "srtp", "rtp",
+              "rtpmanager", "videotestsrc", "audiotestsrc"]
+    missing = list(filter(lambda p: Gst.Registry.get().find_plugin(p) is None, needed))
+    if len(missing):
+        print('Missing gstreamer plugins:', missing)
+        return False
+    return True

raspberry/roberto/views/websocket/routes.py

@@ -55,6 +55,8 @@ def applyDeadZone(value, threshold):
 
 # https://pfertyk.me/2020/03/webrtc-a-working-example/
 
+from roberto import webrtccamera
+
 ROOM = 'default'
 
 @websocket_blueprint.route('/camera')
@@ -66,6 +68,18 @@ def webrtc_message(data):
     sid = request.sid
     print('Message from {}: {}'.format(sid, data))
     socketio.emit('data', data=data, room=ROOM, namespace='/webrtc', skip_sid=sid)
+    if 'type' in data:
+        if data['type'] == 'offer':
+            print("OFFER")
+
+        elif data['type'] == 'answer':
+            print("ANSWER")
+            if 'sdp' in data:
+                webrtccamera.handle_sdp_answer(data['sdp'])
+        elif data['type'] == 'candidate':
+            print("CANDIDATE")
+            if 'candidiate' in data:
+                webrtccamera.handle_ice(data)
 
 @socketio.on('disconnect', namespace='/webrtc')
 def disconnect():
@@ -79,4 +93,6 @@ def connect():
     print("Received Connect message from %s" % sid)
     socketio.emit('ready', room=ROOM, namespace='/webrtc', skip_sid=sid)
     join_room(ROOM)
+    if not webrtccamera.connected:
+        webrtccamera.connect(socketio, ROOM, 'gstwebrtc1000')
 

raspberry/roberto/views/websocket/templates/camera.html

@@ -36,7 +36,8 @@
      
 socket.on('data', (data) => {
   console.log('Data received: ',data);
-  handleSignalingData(data);
+  var json_data = JSON.parse(data);
+  handleSignalingData(json_data);
 });
 
 socket.on('ready', () => {
@@ -129,6 +130,9 @@ let handleSignalingData = (data) => {
     case 'candidate':
       pc.addIceCandidate(new RTCIceCandidate(data.candidate));
       break;
+    default:
+      console.log("got unknown message of type: ", data.type);
+      break;
   }
 };