diff --git a/README.md b/README.md index ef37a8d..9d09d9e 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,14 @@ sudo apt install gstreamer1.0-tools Sender (Pi): gst-launch-1.0 -e v4l2src do-timestamp=true ! video/x-h264,width=640,height=480,framerate=30/1 ! h264parse ! rtph264pay config-interval=1 ! gdppay ! udpsink host=192.168.178.20 port=5000 Receiver: gst-launch-1.0 -v udpsrc port=5000 ! gdpdepay ! rtph264depay ! avdec_h264 ! fpsdisplaysink sync=false text-overlay=false + +## documentation + +https://flask-socketio.readthedocs.io/en/latest/ + +https://github.com/pfertyk/webrtc-working-example + +https://github.com/nanomosfet/WebRTC-Flask-server/blob/master/webRTCserver/webRTCserver.py + +http://blog.nirbheek.in/2018/02/gstreamer-webrtc.html +https://gitlab.freedesktop.org/gstreamer/gst-examples/-/tree/master/webrtc/sendrecv/gst diff --git a/raspberry/requirements.py b/raspberry/requirements.py index 01cc378..2c0cae7 100755 --- a/raspberry/requirements.py +++ b/raspberry/requirements.py @@ -8,3 +8,6 @@ python3-picamera libjs-jquery libjs-bootstrap python3-eventlet + +gir1.2-gst-plugins-bad-1.0 +python3-gst-1.0 diff --git a/raspberry/roberto/__init__.py b/raspberry/roberto/__init__.py index c535229..9b76865 100644 --- a/raspberry/roberto/__init__.py +++ b/raspberry/roberto/__init__.py @@ -18,6 +18,8 @@ login.login_view = "users.login" socketio = SocketIO() from roberto.camera.camera_opencv import Camera camera = Camera() +from roberto.camera.camera_gstreamer_webrtc import WebRTCCamera +webrtccamera = WebRTCCamera(1000, 1001) from roberto.Serial import Serial serial = Serial() diff --git a/raspberry/roberto/camera/camera_gstreamer_webrtc.py b/raspberry/roberto/camera/camera_gstreamer_webrtc.py new file mode 100644 index 0000000..173db10 --- /dev/null +++ b/raspberry/roberto/camera/camera_gstreamer_webrtc.py @@ -0,0 +1,150 @@ +import os +import sys + +import gi +gi.require_version('Gst', '1.0') +from gi.repository import Gst +gi.require_version('GstWebRTC', '1.0') +from gi.repository import GstWebRTC +gi.require_version('GstSdp', '1.0') +from gi.repository import GstSdp + +import json + +PIPELINE_DESC = ''' +webrtcbin name=sendrecv bundle-policy=max-bundle stun-server=stun://stun.l.google.com:19302 + videotestsrc is-live=true pattern=ball ! videoconvert ! queue ! vp8enc deadline=1 ! rtpvp8pay ! + queue ! application/x-rtp,media=video,encoding-name=VP8,payload=97 ! sendrecv. + audiotestsrc is-live=true wave=red-noise ! audioconvert ! audioresample ! queue ! opusenc !rtpopuspay ! + queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 ! sendrecv. +''' + +class WebRTCCamera: + def __init__(self, id_, peer_id, server=None): + self.id_ = id_ + self.conn = None + self.pipe = None + self.webrtc = None + self.socketio = None + self.peer_id = peer_id + self.room = 'default' + self.sid = 'gstwebrtc1000' + self.server = server or 'wss://localhost:5000/webrtc' + self.connected = False + + Gst.init(None) + if not check_plugins(): + sys.exit(1) + + def connect(self, socketio, room, sid): + self.socketio = socketio + self.room = room + self.sid = sid + self.start_pipeline() + self.connected = True + + def disconnect(self): + self.connected = False + self.close_pipeline() + self.socketio = None + + def start_pipeline(self): + self.pipe = Gst.parse_launch(PIPELINE_DESC) + self.webrtc = self.pipe.get_by_name('sendrecv') + self.webrtc.connect('on-negotiation-needed', self.on_negotiation_needed) + self.webrtc.connect('on-ice-candidate', self.send_ice_candidate_message) + self.webrtc.connect('pad-added', self.on_incoming_stream) + self.pipe.set_state(Gst.State.PLAYING) + + def close_pipeline(self): + self.pipe.set_state(Gst.State.NULL) + self.pipe = None + self.webrtc = None + + def handle_sdp_answer(self, sdp): + print ('Received answer:\n%s' % sdp) + res, sdpmsg = GstSdp.SDPMessage.new() + GstSdp.sdp_message_parse_buffer(bytes(sdp.encode()), sdpmsg) + answer = GstWebRTC.WebRTCSessionDescription.new(GstWebRTC.WebRTCSDPType.ANSWER, sdpmsg) + promise = Gst.Promise.new() + self.webrtc.emit('set-remote-description', answer, promise) + promise.interrupt() + + def handle_ice(self, ice): + candidate = ice['candidate'] + sdpmlineindex = ice['sdpMLineIndex'] + self.webrtc.emit('add-ice-candidate', sdpmlineindex, candidate) + + def on_negotiation_needed(self, element): + promise = Gst.Promise.new_with_change_func(self.on_offer_created, element, None) + element.emit('create-offer', None, promise) + + def send_ice_candidate_message(self, _, mlineindex, candidate): + icemsg = json.dumps({'type': 'candidate', 'candidate': {'candidate': candidate, 'sdpMLineIndex': mlineindex}}) + self.socketio.emit('data', type='candidate', data=icemsg, room=self.room, namespace='/webrtc', skip_sid=self.sid) + + def on_incoming_stream(self, _, pad): + if pad.direction != Gst.PadDirection.SRC: + return + + decodebin = Gst.ElementFactory.make('decodebin') + decodebin.connect('pad-added', self.on_incoming_decodebin_stream) + self.pipe.add(decodebin) + decodebin.sync_state_with_parent() + self.webrtc.link(decodebin) + + def on_incoming_decodebin_stream(self, _, pad): + if not pad.has_current_caps(): + print (pad, 'has no caps, ignoring') + return + + caps = pad.get_current_caps() + assert (len(caps)) + s = caps[0] + name = s.get_name() + if name.startswith('video'): + q = Gst.ElementFactory.make('queue') + conv = Gst.ElementFactory.make('videoconvert') + sink = Gst.ElementFactory.make('autovideosink') + self.pipe.add(q, conv, sink) + self.pipe.sync_children_states() + pad.link(q.get_static_pad('sink')) + q.link(conv) + conv.link(sink) + elif name.startswith('audio'): + q = Gst.ElementFactory.make('queue') + conv = Gst.ElementFactory.make('audioconvert') + resample = Gst.ElementFactory.make('audioresample') + sink = Gst.ElementFactory.make('autoaudiosink') + self.pipe.add(q, conv, resample, sink) + self.pipe.sync_children_states() + pad.link(q.get_static_pad('sink')) + q.link(conv) + conv.link(resample) + resample.link(sink) + + def on_offer_created(self, promise, _, __): + promise.wait() + reply = promise.get_reply() + offer = reply['offer'] + promise = Gst.Promise.new() + self.webrtc.emit('set-local-description', offer, promise) + promise.interrupt() + text = offer.sdp.as_text() + print ('Sending offer:\n%s' % text) + msg = json.dumps({'type': 'offer', 'sdp': text}) + self.socketio.emit('data', type='offer', data=msg, room=self.room, namespace='/webrtc', skip_sid=self.sid) + + + + + + +def check_plugins(): + needed = ["opus", "vpx", "nice", "webrtc", "dtls", "srtp", "rtp", + "rtpmanager", "videotestsrc", "audiotestsrc"] + missing = list(filter(lambda p: Gst.Registry.get().find_plugin(p) is None, needed)) + if len(missing): + print('Missing gstreamer plugins:', missing) + return False + return True diff --git a/raspberry/roberto/views/websocket/routes.py b/raspberry/roberto/views/websocket/routes.py index 3328a87..f24b263 100644 --- a/raspberry/roberto/views/websocket/routes.py +++ b/raspberry/roberto/views/websocket/routes.py @@ -55,6 +55,8 @@ def applyDeadZone(value, threshold): # https://pfertyk.me/2020/03/webrtc-a-working-example/ +from roberto import webrtccamera + ROOM = 'default' @websocket_blueprint.route('/camera') @@ -66,6 +68,18 @@ def webrtc_message(data): sid = request.sid print('Message from {}: {}'.format(sid, data)) socketio.emit('data', data=data, room=ROOM, namespace='/webrtc', skip_sid=sid) + if 'type' in data: + if data['type'] == 'offer': + print("OFFER") + + elif data['type'] == 'answer': + print("ANSWER") + if 'sdp' in data: + webrtccamera.handle_sdp_answer(data['sdp']) + elif data['type'] == 'candidate': + print("CANDIDATE") + if 'candidiate' in data: + webrtccamera.handle_ice(data) @socketio.on('disconnect', namespace='/webrtc') def disconnect(): @@ -79,4 +93,6 @@ def connect(): print("Received Connect message from %s" % sid) socketio.emit('ready', room=ROOM, namespace='/webrtc', skip_sid=sid) join_room(ROOM) + if not webrtccamera.connected: + webrtccamera.connect(socketio, ROOM, 'gstwebrtc1000') diff --git a/raspberry/roberto/views/websocket/templates/camera.html b/raspberry/roberto/views/websocket/templates/camera.html index a72304e..f78725c 100644 --- a/raspberry/roberto/views/websocket/templates/camera.html +++ b/raspberry/roberto/views/websocket/templates/camera.html @@ -36,7 +36,8 @@ socket.on('data', (data) => { console.log('Data received: ',data); - handleSignalingData(data); + var json_data = JSON.parse(data); + handleSignalingData(json_data); }); socket.on('ready', () => { @@ -129,6 +130,9 @@ let handleSignalingData = (data) => { case 'candidate': pc.addIceCandidate(new RTCIceCandidate(data.candidate)); break; + default: + console.log("got unknown message of type: ", data.type); + break; } };