Browse Source

webrtc test

split-pipe
Hendrik Langer 4 years ago
parent
commit
8e342f4a5e
  1. 11
      README.md
  2. 3
      raspberry/requirements.py
  3. 2
      raspberry/roberto/__init__.py
  4. 150
      raspberry/roberto/camera/camera_gstreamer_webrtc.py
  5. 16
      raspberry/roberto/views/websocket/routes.py
  6. 6
      raspberry/roberto/views/websocket/templates/camera.html

11
README.md

@ -37,3 +37,14 @@ sudo apt install gstreamer1.0-tools
Sender (Pi): gst-launch-1.0 -e v4l2src do-timestamp=true ! video/x-h264,width=640,height=480,framerate=30/1 ! h264parse ! rtph264pay config-interval=1 ! gdppay ! udpsink host=192.168.178.20 port=5000
Receiver: gst-launch-1.0 -v udpsrc port=5000 ! gdpdepay ! rtph264depay ! avdec_h264 ! fpsdisplaysink sync=false text-overlay=false
## documentation
https://flask-socketio.readthedocs.io/en/latest/
https://github.com/pfertyk/webrtc-working-example
https://github.com/nanomosfet/WebRTC-Flask-server/blob/master/webRTCserver/webRTCserver.py
http://blog.nirbheek.in/2018/02/gstreamer-webrtc.html
https://gitlab.freedesktop.org/gstreamer/gst-examples/-/tree/master/webrtc/sendrecv/gst

3
raspberry/requirements.py

@ -8,3 +8,6 @@ python3-picamera
libjs-jquery
libjs-bootstrap
python3-eventlet
gir1.2-gst-plugins-bad-1.0
python3-gst-1.0

2
raspberry/roberto/__init__.py

@ -18,6 +18,8 @@ login.login_view = "users.login"
socketio = SocketIO()
from roberto.camera.camera_opencv import Camera
camera = Camera()
from roberto.camera.camera_gstreamer_webrtc import WebRTCCamera
webrtccamera = WebRTCCamera(1000, 1001)
from roberto.Serial import Serial
serial = Serial()

150
raspberry/roberto/camera/camera_gstreamer_webrtc.py

@ -0,0 +1,150 @@
import os
import sys
import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst
gi.require_version('GstWebRTC', '1.0')
from gi.repository import GstWebRTC
gi.require_version('GstSdp', '1.0')
from gi.repository import GstSdp
import json
PIPELINE_DESC = '''
webrtcbin name=sendrecv bundle-policy=max-bundle stun-server=stun://stun.l.google.com:19302
videotestsrc is-live=true pattern=ball ! videoconvert ! queue ! vp8enc deadline=1 ! rtpvp8pay !
queue ! application/x-rtp,media=video,encoding-name=VP8,payload=97 ! sendrecv.
audiotestsrc is-live=true wave=red-noise ! audioconvert ! audioresample ! queue ! opusenc !rtpopuspay !
queue ! application/x-rtp,media=audio,encoding-name=OPUS,payload=96 ! sendrecv.
'''
class WebRTCCamera:
def __init__(self, id_, peer_id, server=None):
self.id_ = id_
self.conn = None
self.pipe = None
self.webrtc = None
self.socketio = None
self.peer_id = peer_id
self.room = 'default'
self.sid = 'gstwebrtc1000'
self.server = server or 'wss://localhost:5000/webrtc'
self.connected = False
Gst.init(None)
if not check_plugins():
sys.exit(1)
def connect(self, socketio, room, sid):
self.socketio = socketio
self.room = room
self.sid = sid
self.start_pipeline()
self.connected = True
def disconnect(self):
self.connected = False
self.close_pipeline()
self.socketio = None
def start_pipeline(self):
self.pipe = Gst.parse_launch(PIPELINE_DESC)
self.webrtc = self.pipe.get_by_name('sendrecv')
self.webrtc.connect('on-negotiation-needed', self.on_negotiation_needed)
self.webrtc.connect('on-ice-candidate', self.send_ice_candidate_message)
self.webrtc.connect('pad-added', self.on_incoming_stream)
self.pipe.set_state(Gst.State.PLAYING)
def close_pipeline(self):
self.pipe.set_state(Gst.State.NULL)
self.pipe = None
self.webrtc = None
def handle_sdp_answer(self, sdp):
print ('Received answer:\n%s' % sdp)
res, sdpmsg = GstSdp.SDPMessage.new()
GstSdp.sdp_message_parse_buffer(bytes(sdp.encode()), sdpmsg)
answer = GstWebRTC.WebRTCSessionDescription.new(GstWebRTC.WebRTCSDPType.ANSWER, sdpmsg)
promise = Gst.Promise.new()
self.webrtc.emit('set-remote-description', answer, promise)
promise.interrupt()
def handle_ice(self, ice):
candidate = ice['candidate']
sdpmlineindex = ice['sdpMLineIndex']
self.webrtc.emit('add-ice-candidate', sdpmlineindex, candidate)
def on_negotiation_needed(self, element):
promise = Gst.Promise.new_with_change_func(self.on_offer_created, element, None)
element.emit('create-offer', None, promise)
def send_ice_candidate_message(self, _, mlineindex, candidate):
icemsg = json.dumps({'type': 'candidate', 'candidate': {'candidate': candidate, 'sdpMLineIndex': mlineindex}})
self.socketio.emit('data', type='candidate', data=icemsg, room=self.room, namespace='/webrtc', skip_sid=self.sid)
def on_incoming_stream(self, _, pad):
if pad.direction != Gst.PadDirection.SRC:
return
decodebin = Gst.ElementFactory.make('decodebin')
decodebin.connect('pad-added', self.on_incoming_decodebin_stream)
self.pipe.add(decodebin)
decodebin.sync_state_with_parent()
self.webrtc.link(decodebin)
def on_incoming_decodebin_stream(self, _, pad):
if not pad.has_current_caps():
print (pad, 'has no caps, ignoring')
return
caps = pad.get_current_caps()
assert (len(caps))
s = caps[0]
name = s.get_name()
if name.startswith('video'):
q = Gst.ElementFactory.make('queue')
conv = Gst.ElementFactory.make('videoconvert')
sink = Gst.ElementFactory.make('autovideosink')
self.pipe.add(q, conv, sink)
self.pipe.sync_children_states()
pad.link(q.get_static_pad('sink'))
q.link(conv)
conv.link(sink)
elif name.startswith('audio'):
q = Gst.ElementFactory.make('queue')
conv = Gst.ElementFactory.make('audioconvert')
resample = Gst.ElementFactory.make('audioresample')
sink = Gst.ElementFactory.make('autoaudiosink')
self.pipe.add(q, conv, resample, sink)
self.pipe.sync_children_states()
pad.link(q.get_static_pad('sink'))
q.link(conv)
conv.link(resample)
resample.link(sink)
def on_offer_created(self, promise, _, __):
promise.wait()
reply = promise.get_reply()
offer = reply['offer']
promise = Gst.Promise.new()
self.webrtc.emit('set-local-description', offer, promise)
promise.interrupt()
text = offer.sdp.as_text()
print ('Sending offer:\n%s' % text)
msg = json.dumps({'type': 'offer', 'sdp': text})
self.socketio.emit('data', type='offer', data=msg, room=self.room, namespace='/webrtc', skip_sid=self.sid)
def check_plugins():
needed = ["opus", "vpx", "nice", "webrtc", "dtls", "srtp", "rtp",
"rtpmanager", "videotestsrc", "audiotestsrc"]
missing = list(filter(lambda p: Gst.Registry.get().find_plugin(p) is None, needed))
if len(missing):
print('Missing gstreamer plugins:', missing)
return False
return True

16
raspberry/roberto/views/websocket/routes.py

@ -55,6 +55,8 @@ def applyDeadZone(value, threshold):
# https://pfertyk.me/2020/03/webrtc-a-working-example/
from roberto import webrtccamera
ROOM = 'default'
@websocket_blueprint.route('/camera')
@ -66,6 +68,18 @@ def webrtc_message(data):
sid = request.sid
print('Message from {}: {}'.format(sid, data))
socketio.emit('data', data=data, room=ROOM, namespace='/webrtc', skip_sid=sid)
if 'type' in data:
if data['type'] == 'offer':
print("OFFER")
elif data['type'] == 'answer':
print("ANSWER")
if 'sdp' in data:
webrtccamera.handle_sdp_answer(data['sdp'])
elif data['type'] == 'candidate':
print("CANDIDATE")
if 'candidiate' in data:
webrtccamera.handle_ice(data)
@socketio.on('disconnect', namespace='/webrtc')
def disconnect():
@ -79,4 +93,6 @@ def connect():
print("Received Connect message from %s" % sid)
socketio.emit('ready', room=ROOM, namespace='/webrtc', skip_sid=sid)
join_room(ROOM)
if not webrtccamera.connected:
webrtccamera.connect(socketio, ROOM, 'gstwebrtc1000')

6
raspberry/roberto/views/websocket/templates/camera.html

@ -36,7 +36,8 @@
socket.on('data', (data) => {
console.log('Data received: ',data);
handleSignalingData(data);
var json_data = JSON.parse(data);
handleSignalingData(json_data);
});
socket.on('ready', () => {
@ -129,6 +130,9 @@ let handleSignalingData = (data) => {
case 'candidate':
pc.addIceCandidate(new RTCIceCandidate(data.candidate));
break;
default:
console.log("got unknown message of type: ", data.type);
break;
}
};

Loading…
Cancel
Save