Django: Subprocess Continuous Output To HTML View

Question

I need a HTML webpage in my Django app to load and show the continous output of a script in a scrollable box. Is this possible?

I\'m presently using a subprocess

Answer 1

You could simplify your task using StreamingHttpResponse and Popen:

def test_iterator():
    from subprocess import Popen, PIPE, CalledProcessError

    with Popen(['ping', 'localhost'], stdout=PIPE, bufsize=1, universal_newlines=True) as p:
        for line in p.stdout:
            yield(line + '<br>') # process line here

    if p.returncode != 0:
        raise CalledProcessError(p.returncode, p.args)

def busy_view(request):
    from django.http import StreamingHttpResponse
    return StreamingHttpResponse(test_iterator())

StreamingHttpResponse expects an iterator as its parameter. An iterator function is one which has a yield expression (or a generator expression), and its return value is a generator object (an iterator).

In this example, I simply echo the ping command to prove it works.

Substitute ['ping', 'localhost'] by a list (it has to be a list if you pass parameters to the command - in this case, localhost). Your original ['python', script] should work.

If you want to know more about generators, I would recommend Trey Hunner's talk, and also strongly that you read chapter 14 of Fluent Python book. Both are amazing sources.

Disclaimer:

Performance considerations

Django is designed for short-lived requests. Streaming responses will tie a worker process for the entire duration of the response. This may result in poor performance.

Generally speaking, you should perform expensive tasks outside of the request-response cycle, rather than resorting to a streamed response.

Answer 2

What you would want is websockets, or Channels as they're known in Django.

https://channels.readthedocs.io/en/latest/

This allows you to send messages from the backend to the frontend without having to pull the messages on the frontend or reload the page.

Something worth mention is that you could also stream the output to multiple clients and also send back commands to your backend.

Approach tailored to your code

Please notice, this is untested as I do not have access to your code and therefor you might need some minor adjustments, I believe however the provided code should illustrate the concept.

Settings.py

INSTALLED_APPS = (
#Other installed Apps
       'Channels',
)
CHANNEL_LAYERS = {
      "default": {
          "BACKEND": "asgiref.inmemory.ChannelLayer",
            "ROUTING": "django_channels.routing.channel_routing",
      },
}

routing.py (add file in same folder as settings.py)

from django_channels_app.consumers import message_ws, listener_add, listener_discconect

channel_routing = [
      route("websocket.receive", message_ws),
      route("websocket.disconnect", listener_discconect),
      route("websocket.connect", listener_add),
]

In your module:

import threading
from channels import Group

class PreserializeThread(threading.Thread):
    def __init__(self, request, *args, **kwargs):
        self.request = request
        super(PreserializeThread, self).__init__(*args, **kwargs)

    def run(self):
        GenerateProjectConfig(request)
        home = os.getcwd()
        project_id = request.session['projectname']
        staging_folder = home + "/staging/" + project_id + "/"
        output = ""
        os.chdir(staging_folder)
        script = home + '/webscripts/terraformdeploy.py'
        try:
            output = subprocess.check_output(['python', script], shell=True)
            Group("django_channels_group").send({
                "text": output,
            })

            # NOTICE THIS WILL BLOCK; 
            # You could try the following, untested snippet


#    proc = subprocess.Popen(['python', script], shell=True, #stdout=subprocess.PIPE)
#    
#    line = proc.stdout.readline()
#    while line:
#        line = proc.stdout.readline()
#        Group("django_channels_group").send({
#                        "text": line,
#                    })
#    Group("django_channels_group").send({
#        "text": "Finished",
#    })
        except subprocess.CalledProcessError:
            exit_code, error_msg = (
                output.returncode,output.output)
        os.chdir(home)

def listener_add(message):
    Group("django_channels_group").add(
        message.reply_channel)

def listener_discconect(message):
    Group("django_channels_group").discard(
        message.reply_channel)

def message_ws(message):
    Group("django_channels_group").send({
          "text": "My group message",
     })

def projectprogress(request):
    ProgressThread(request).start()
    return render(request, 'projectprogress.html', locals())

html

<style>
  div.ex1 {
  background-color: black;
  width: 900px;
  height: 500px;
  overflow: scroll;
  margin: 50px;
}
</style>

<body style="background-color: #565c60; font-family: Georgia, 'Times New Roman', Times, serif; color: white; margin:0"></body>
    <div id="output">
        
    </div>
    <div class="container">
        <a class="button button--wide button--white" href="home.html" title="Home" style="color: white; margin: 60px;">
            <span class="button__inner">
          Home
        </span>
        </a>
    </div>
</body>
</html>

<script>
socket = new WebSocket("ws://127.0.0.1:8000/"); #Or your server IP address
socket.onmessage = function(e) {
    const data = JSON.parse(e.data);
    document.querySelector('#ouput').value += (data.message + '\n');
}
socket.onopen = function() {
    socket.send("Test message");
}
</script>

More generic answer

Backend:

chat/consumers.py:

import json
from asgiref.sync import async_to_sync
from channels.generic.websocket import WebsocketConsumer

class ChatConsumer(WebsocketConsumer):
    def connect(self):
        self.room_name = self.scope['url_route']['kwargs']['room_name']
        self.room_group_name = 'chat_%s' % self.room_name

        # Join room group
        async_to_sync(self.channel_layer.group_add)(
            self.room_group_name,
            self.channel_name
        )

        self.accept()

    def disconnect(self, close_code):
        # Leave room group
        async_to_sync(self.channel_layer.group_discard)(
            self.room_group_name,
            selfreturn render(request, 'projectprogress.html', locals()).channel_name
        )

    def send_message(self, event):
        message = event['message']

        # Send message to WebSocket
        self.send(text_data=json.dumps({
            'message': message
        }))

mysite/settings.py:

# Channels
ASGI_APPLICATION = 'mysite.routing.application'
CHANNEL_LAYERS = {
    'default': {
        'BACKEND': 'channels_redis.core.RedisChannelLayer',
        'CONFIG': {
            "hosts": [('127.0.0.1', 6379)],
        },
    },
}

mysite/routing.py:

from channels.auth import AuthMiddlewareStack
from channels.routing import ProtocolTypeRouter, URLRouter
import chat.routing

application = ProtocolTypeRouter({
    # (http->django views is added by default)
    'websocket': AuthMiddlewareStack(
        URLRouter(
            chat.routing.websocket_urlpatterns
        )
    ),
})

chat/routing.py:

from django.urls import re_path

from . import consumers

websocket_urlpatterns = [
    re_path(r'ws/chat/(?P<room_name>\w+)/$', consumers.ChatConsumer),
]

Frontend:

<script>
const chatSocket = new WebSocket(
            'ws://'
            + window.location.host
            + '/ws/chat/'
            + roomName
            + '/'
        );

        chatSocket.onmessage = function(e) {
            const data = JSON.parse(e.data);
            document.querySelector('#chat-log').value += (data.message + '\n');
        };
</script>